From d727a0f0ae0d59a30faf736c9378fe4574de41e4 Mon Sep 17 00:00:00 2001 From: Brendan Ryan <1572504+brendanjryan@users.noreply.github.com> Date: Sun, 18 Oct 2026 17:07:32 +0000 Subject: [PATCH 1/2] fix: serve Markdown for every TIP Co-authored-by: Derek Cofausper <256792747+decofe@users.noreply.github.com> --- .github/workflows/main.yml | 8 ++ .gitignore | 3 +- README.md | 31 +++++++ package.json | 9 +- pnpm-lock.yaml | 55 ++++++++++++ scripts/check-markdown.ts | 56 ++++++++++++ scripts/sync.ts | 11 +++ src/lib/MarkdownExport.test.ts | 110 ++++++++++++++++++++++++ src/lib/MarkdownExport.ts | 93 ++++++++++++++++++++ src/lib/MarkdownHttp.test.ts | 153 +++++++++++++++++++++++++++++++++ src/lib/MarkdownHttp.ts | 142 ++++++++++++++++++++++++++++++ src/lib/MdRouter.ts | 121 ++------------------------ src/lib/TipStore.ts | 28 ++++++ src/lib/Tips.fns.ts | 37 +------- src/worker.tsx | 21 ++++- vitest.config.ts | 4 + 16 files changed, 726 insertions(+), 156 deletions(-) create mode 100644 scripts/check-markdown.ts create mode 100644 src/lib/MarkdownExport.test.ts create mode 100644 src/lib/MarkdownExport.ts create mode 100644 src/lib/MarkdownHttp.test.ts create mode 100644 src/lib/MarkdownHttp.ts create mode 100644 src/lib/TipStore.ts create mode 100644 vitest.config.ts diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 22b3632..f9525c1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,5 +1,6 @@ name: Main on: + pull_request: push: branches: [main] jobs: @@ -18,7 +19,11 @@ jobs: - name: Typecheck run: pnpm check:types + - name: Test + run: pnpm test + deploy: + if: github.event_name == 'push' && github.ref == 'refs/heads/main' needs: check runs-on: ubuntu-latest steps: @@ -51,3 +56,6 @@ jobs: with: command: deploy apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} + + - name: Verify deployed Markdown + run: pnpm check:markdown --url https://tips.sh --manifest .artifacts/markdown-manifest.json diff --git a/.gitignore b/.gitignore index e4e0a7e..ca1a12d 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,5 @@ todos.json !.env.example .env public/robots.txt -worker-configuration.d.ts \ No newline at end of file +worker-configuration.d.ts +.artifacts diff --git a/README.md b/README.md index 8315dc9..e143a13 100644 --- a/README.md +++ b/README.md @@ -1 +1,32 @@ # tips.sh + +## Markdown + +Every TIP page has a `/.md` endpoint, including proposal variants such as +`/1061-1.md` and title-based slugs. These URLs always return Markdown, regardless +of user agent. `GET /` with `Accept: text/markdown` returns the same content; +normal browser requests and link-preview bots retain HTML. Negotiated URLs use +`Vary: Accept, User-Agent` and `Cache-Control: private, no-store` because the +Cloudflare default cache key does not distinguish those headers. Explicit `.md` +URLs can be cached independently. Unknown Markdown IDs return 404, and database +failures return 503 rather than falling back to HTML. + +`/llms.txt` and `/index.md` list the complete website inventory with links to each +Markdown page and the originating PR for proposals. Exports preserve frontmatter, +code, tables, math, and Mermaid source. Relative TIP links resolve to the matching +proposal revision where available; other relative assets resolve against the +source repository or PR. + +Run `pnpm test` for routing, negotiation, content preservation, and inventory +coverage checks. `pnpm sync` also writes `.artifacts/markdown-manifest.json`, +containing expected hashes from that sync. After deploying, verify the public +endpoints against that local file: + +```sh +pnpm check:markdown --url https://tips.sh --manifest .artifacts/markdown-manifest.json +``` + +The deployment workflow runs this check after publishing. Generate the expected +manifest from the intended sync; do not substitute a manifest downloaded from +the deployed site. A concurrent content sync can require rerunning the sync and +verification together. diff --git a/package.json b/package.json index 31c5d82..878f9e0 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,6 @@ "name": "tips", "private": true, "type": "module", - "packageManager": "pnpm@10.33.0", "imports": { "#/*": "./src/*" }, @@ -10,6 +9,7 @@ "build": "vp build", "check": "vp check --fix", "check:types": "tsc --noEmit", + "check:markdown": "node --import=tsx scripts/check-markdown.ts", "db:init": "wrangler d1 migrations apply tips --local && pnpm sync", "deploy": "npm run build && wrangler deploy", "dev": "vite dev --port 3000", @@ -18,7 +18,7 @@ "prebuild": "node --import=tsx scripts/gen-robots.ts", "preview": "npm run build && vp preview", "sync": "node --env-file-if-exists=.env --import=tsx scripts/sync.ts", - "test": "vp test run" + "test": "vp test run --config vitest.config.ts" }, "dependencies": { "@cloudflare/puppeteer": "^1.0.6", @@ -41,10 +41,12 @@ "rehype-katex": "^7.0.1", "rehype-raw": "^7.0.0", "rehype-stringify": "^10.0.1", + "remark-frontmatter": "^5.0.0", "remark-gfm": "^4.0.1", "remark-math": "^6.0.0", "remark-parse": "^11.0.0", "remark-rehype": "^11.1.2", + "remark-stringify": "^11.0.0", "shiki": "^4.0.2", "tailwindcss": "^4.1.18", "takumi-js": "^1.0.0-rc.16", @@ -67,5 +69,6 @@ "vite-plus": "0.1.16", "vite-tsconfig-paths": "^5.1.4", "wrangler": "^4.81.1" - } + }, + "packageManager": "pnpm@10.33.0" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b515a75..4444018 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -72,6 +72,9 @@ importers: rehype-stringify: specifier: ^10.0.1 version: 10.0.1 + remark-frontmatter: + specifier: ^5.0.0 + version: 5.0.0 remark-gfm: specifier: ^4.0.1 version: 4.0.1 @@ -84,6 +87,9 @@ importers: remark-rehype: specifier: ^11.1.2 version: 11.1.2 + remark-stringify: + specifier: ^11.0.0 + version: 11.0.0 shiki: specifier: ^4.0.2 version: 4.0.2 @@ -2700,6 +2706,9 @@ packages: fast-fifo@1.3.2: resolution: {integrity: sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==} + fault@2.0.1: + resolution: {integrity: sha512-WtySTkS4OKev5JtpHXnib4Gxiurzh5NCGvWrFaZ34m6JehfTUhKZvn9njTfw48t6JumVQOmrKqpmGcdwxnhqBQ==} + fd-slicer@1.1.0: resolution: {integrity: sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==} @@ -2716,6 +2725,10 @@ packages: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} + format@0.2.2: + resolution: {integrity: sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==} + engines: {node: '>=0.4.x'} + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -3066,6 +3079,9 @@ packages: mdast-util-from-markdown@2.0.3: resolution: {integrity: sha512-W4mAWTvSlKvf8L6J+VN9yLSqQ9AOAAvHuoDAmPkz4dHf553m5gVj2ejadHJhoJmcmxEnOv6Pa8XJhpxE93kb8Q==} + mdast-util-frontmatter@2.0.1: + resolution: {integrity: sha512-LRqI9+wdgC25P0URIJY9vwocIzCcksduHQ9OF2joxQoyTNVduwLAFUzjoopuRJbJAReaKrNQKAZKL3uCMugWJA==} + mdast-util-gfm-autolink-literal@2.0.1: resolution: {integrity: sha512-5HVP2MKaP6L+G6YaxPNjuL0BPrq9orG3TsrZ9YXbA3vDw/ACI4MEsnoDpn6ZNm7GnZgtAcONJyPhOP8tNJQavQ==} @@ -3120,6 +3136,9 @@ packages: micromark-core-commonmark@2.0.3: resolution: {integrity: sha512-RDBrHEMSxVFLg6xvnXmb1Ayr2WzLAWjeSATAoxwKYJV94TeNavgoIdA0a9ytzDSVzBy2YKFK+emCPOEibLeCrg==} + micromark-extension-frontmatter@2.0.0: + resolution: {integrity: sha512-C4AkuM3dA58cgZha7zVnuVxBhDsbttIMiytjgsM2XbHAB2faRVaHRle40558FBN+DJcrLNCoqG5mlrpdU4cRtg==} + micromark-extension-gfm-autolink-literal@2.1.0: resolution: {integrity: sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw==} @@ -3464,6 +3483,9 @@ packages: rehype-stringify@10.0.1: resolution: {integrity: sha512-k9ecfXHmIPuFVI61B9DeLPN0qFHfawM6RsuX48hoqlaKSF61RskNjSm1lI8PhBEM0MRdLxVVm4WmTqJQccH9mA==} + remark-frontmatter@5.0.0: + resolution: {integrity: sha512-XTFYvNASMe5iPN0719nPrdItC9aU0ssC4v14mH1BCi1u0n1gAocqcujWUrByftZTbLhRtiKRyjYTSIOcr69UVQ==} + remark-gfm@4.0.1: resolution: {integrity: sha512-1quofZ2RQ9EWdeN34S79+KExV1764+wCUGop5CPL1WGdD0ocPpu91lzPGbwWMECpEpd42kJGQwzRfyov9j4yNg==} @@ -6273,6 +6295,10 @@ snapshots: fast-fifo@1.3.2: {} + fault@2.0.1: + dependencies: + format: 0.2.2 + fd-slicer@1.1.0: dependencies: pend: 1.2.0 @@ -6285,6 +6311,8 @@ snapshots: dependencies: to-regex-range: 5.0.1 + format@0.2.2: {} + fsevents@2.3.3: optional: true @@ -6702,6 +6730,17 @@ snapshots: transitivePeerDependencies: - supports-color + mdast-util-frontmatter@2.0.1: + dependencies: + '@types/mdast': 4.0.4 + devlop: 1.1.0 + escape-string-regexp: 5.0.0 + mdast-util-from-markdown: 2.0.3 + mdast-util-to-markdown: 2.1.2 + micromark-extension-frontmatter: 2.0.0 + transitivePeerDependencies: + - supports-color + mdast-util-gfm-autolink-literal@2.0.1: dependencies: '@types/mdast': 4.0.4 @@ -6898,6 +6937,13 @@ snapshots: micromark-util-symbol: 2.0.1 micromark-util-types: 2.0.2 + micromark-extension-frontmatter@2.0.0: + dependencies: + fault: 2.0.1 + micromark-util-character: 2.1.1 + micromark-util-symbol: 2.0.1 + micromark-util-types: 2.0.2 + micromark-extension-gfm-autolink-literal@2.1.0: dependencies: micromark-util-character: 2.1.1 @@ -7464,6 +7510,15 @@ snapshots: hast-util-to-html: 9.0.5 unified: 11.0.5 + remark-frontmatter@5.0.0: + dependencies: + '@types/mdast': 4.0.4 + mdast-util-frontmatter: 2.0.1 + micromark-extension-frontmatter: 2.0.0 + unified: 11.0.5 + transitivePeerDependencies: + - supports-color + remark-gfm@4.0.1: dependencies: '@types/mdast': 4.0.4 diff --git a/scripts/check-markdown.ts b/scripts/check-markdown.ts new file mode 100644 index 0000000..3cc8756 --- /dev/null +++ b/scripts/check-markdown.ts @@ -0,0 +1,56 @@ +/** Verify a deployment against a locally generated, trusted inventory. */ +import { readFileSync } from 'node:fs' +import { parseArgs } from 'node:util' +import { hash, type manifest } from '../src/lib/MarkdownExport' + +const { values } = parseArgs({ + options: { + url: { type: 'string' }, + manifest: { type: 'string' }, + }, +}) +if (!values.url || !values.manifest) + throw new Error('Usage: pnpm check:markdown --url --manifest ') +const origin = new URL(values.url) +const expected = JSON.parse(readFileSync(values.manifest, 'utf8')) as Awaited< + ReturnType +> +if (!Array.isArray(expected) || !expected.length) + throw new Error('Expected a nonempty local TIP manifest') + +async function read(path: string, headers: HeadersInit = {}) { + const response = await fetch(new URL(path, origin), { + headers, + signal: AbortSignal.timeout(30_000), + }) + if (response.status !== 200) throw new Error(`${path}: HTTP ${response.status}`) + if (!response.headers.get('content-type')?.includes('text/markdown')) + throw new Error(`${path}: expected text/markdown`) + const content = await response.text() + if (/^\s*(?:])/i.test(content)) throw new Error(`${path}: HTML response`) + return content +} + +const indexes = await Promise.all(['/llms.txt', '/index.md'].map((path) => read(path))) +// Bound requests rather than starting one connection per TIP. +for (let offset = 0; offset < expected.length; offset += 5) { + await Promise.all( + expected.slice(offset, offset + 5).map(async (entry) => { + for (const index of indexes) { + if (!index.includes(`https://tips.sh${entry.path})`)) + throw new Error(`${entry.id}: missing from index`) + } + const explicit = await read(entry.path, { 'User-Agent': 'Mozilla/5.0', Accept: 'text/html' }) + const negotiated = await read(`/${encodeURIComponent(entry.id)}`, { + Accept: 'text/markdown', + 'User-Agent': 'Mozilla/5.0', + }) + if ((await hash(explicit)) !== entry.sha256 || (await hash(negotiated)) !== entry.sha256) { + throw new Error(`${entry.id}: deployed content differs from the expected sync`) + } + }), + ) +} +console.log( + `Verified ${expected.length} TIPs: explicit and negotiated Markdown, hashes, and discovery`, +) diff --git a/scripts/sync.ts b/scripts/sync.ts index bfb8576..8753394 100644 --- a/scripts/sync.ts +++ b/scripts/sync.ts @@ -1,6 +1,7 @@ /** Standalone sync script — run with: npx tsx scripts/sync.ts */ import * as Sync from '../src/lib/Sync' +import * as MarkdownExport from '../src/lib/MarkdownExport' async function main() { const token = process.env.GITHUB_TOKEN @@ -50,6 +51,16 @@ async function main() { stdio: 'inherit', }) + // Keep the expected hashes outside the deployment: a stale remote manifest + // must not be able to validate a stale deployment. + const { mkdirSync } = await import('fs') + mkdirSync('.artifacts', { recursive: true }) + const inventory = allTips.map((tip) => ({ ...tip, pr_json: tip.prJson })) + writeFileSync( + '.artifacts/markdown-manifest.json', + JSON.stringify(await MarkdownExport.manifest(inventory), null, 2), + ) + console.log('\n✅ Sync complete!') } diff --git a/src/lib/MarkdownExport.test.ts b/src/lib/MarkdownExport.test.ts new file mode 100644 index 0000000..b9a55a8 --- /dev/null +++ b/src/lib/MarkdownExport.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it } from 'vite-plus/test' +import { unified } from 'unified' +import remarkParse from 'remark-parse' +import remarkGfm from 'remark-gfm' +import remarkMath from 'remark-math' +import remarkFrontmatter from 'remark-frontmatter' +import { render, type Tip } from './MarkdownExport' + +const tip: Tip = { + number: '1034', + title: 'Channels', + filename: 'tip-1034.md', + status: 'Draft', + pr_json: '', + content: '', +} + +describe('Markdown exports', () => { + it('preserves frontmatter, tables, code, math, and Mermaid semantically', () => { + const content = [ + '---', + 'id: TIP-1034', + 'title: Channels', + 'authors: Alice', + '---', + '', + '# Channels', + '', + '**Protocol Version**: T3', + '', + '| Name | Value |', + '| --- | --- |', + '| channel | `balance` |', + '', + '```solidity', + '// [Example](./tip-1034.md)', + 'uint256 amount = 1;', + '```', + '', + '$$\\sum_{i=1}^{n} i$$', + '', + '```mermaid', + 'graph LR', + ' A --> B', + '```', + '', + ].join('\n') + const parser = unified().use(remarkParse).use(remarkGfm).use(remarkMath).use(remarkFrontmatter) + const withoutPositions = (value: unknown) => + JSON.parse(JSON.stringify(value, (key, item) => (key === 'position' ? undefined : item))) + expect(withoutPositions(parser.parse(render({ ...tip, content }, [tip])))).toEqual( + withoutPositions(parser.parse(content)), + ) + }) + + it('resolves links, reference definitions, images and HTML while leaving code alone', () => { + const content = [ + '[Other](./tip-1000.md?view=full#specification)', + '[Ref][other]', + '', + '[other]: tip-1000.md#abstract', + '', + '![Diagram](./assets/diagram.svg)', + 'Other', + '', + '', + '`[Example](./tip-1000.md)`', + '', + '[External](https://example.com/tip-1000.md)', + '[Local](#abstract)', + '', + ].join('\n') + const output = render({ ...tip, content }, [ + tip, + { ...tip, number: '1000', filename: 'tip-1000.md' }, + ]) + expect(output).toContain('(https://tips.sh/1000.md?view=full#specification)') + expect(output).toContain('[other]: https://tips.sh/1000.md#abstract') + expect(output).toContain( + '(https://github.com/tempoxyz/tempo/raw/refs/heads/main/tips/assets/diagram.svg)', + ) + expect(output).toContain('href="https://tips.sh/1000.md"') + expect(output).toContain( + 'src="https://github.com/tempoxyz/tempo/raw/refs/heads/main/tips/assets/diagram.svg"', + ) + expect(output).toContain('`[Example](./tip-1000.md)`') + expect(output).toContain('(https://example.com/tip-1000.md)') + expect(output).toContain('(#abstract)') + }) + + it('prefers matching PR variants and uses PR source assets, including for forks', () => { + const proposal = { + ...tip, + number: '1034-1', + pr_json: JSON.stringify({ number: 42 }), + content: '[Other](./tip-1000.md)\n\n![Diagram](./diagram.svg)\n', + } + const inventory = [ + tip, + proposal, + { ...tip, number: '1000', filename: 'tip-1000.md' }, + { ...proposal, number: '1000-1', filename: 'tip-1000.md' }, + ] + const output = render(proposal, inventory) + expect(output).toContain('(https://tips.sh/1000-1.md)') + expect(output).toContain( + '(https://github.com/tempoxyz/tempo/raw/refs/pull/42/head/tips/diagram.svg)', + ) + }) +}) diff --git a/src/lib/MarkdownExport.ts b/src/lib/MarkdownExport.ts new file mode 100644 index 0000000..731fd6e --- /dev/null +++ b/src/lib/MarkdownExport.ts @@ -0,0 +1,93 @@ +/** Export the authored content, resolving links without going through HTML. */ +import { unified } from 'unified' +import remarkParse from 'remark-parse' +import remarkGfm from 'remark-gfm' +import remarkMath from 'remark-math' +import remarkFrontmatter from 'remark-frontmatter' +import remarkStringify from 'remark-stringify' +import { visit } from 'unist-util-visit' +import type { Summary } from './TipStore' + +export type Tip = Summary & { content: string } + +export function pagePath(id: string) { + return `/${encodeURIComponent(id)}.md` +} + +export function render(tip: Tip, inventory: Summary[]): string { + const processor = unified() + .use(remarkParse) + .use(remarkGfm) + .use(remarkMath) + .use(remarkFrontmatter) + .use(remarkStringify) + const tree = processor.parse(tip.content) + const pr = tip.pr_json ? (JSON.parse(tip.pr_json) as { number: number; url: string }) : undefined + // Resolve source-relative assets to the correct proposal, including fork PRs. + const source = pr + ? `https://github.com/tempoxyz/tempo/raw/refs/pull/${pr.number}/head/tips/${tip.filename}` + : `https://github.com/tempoxyz/tempo/raw/refs/heads/main/tips/${tip.filename}` + + const resolve = (url: string) => { + if (!url || /^(?:[a-z][a-z\d+.-]*:|\/\/|#)/i.test(url)) return url + const file = url.match(/^(?:\.\/|\/)?(tip-[^/?#]+\.md)([?#].*)?$/) + if (file) { + const candidates = inventory.filter((entry) => entry.filename === file[1]) + // Prefer the matching PR's revision, then the canonical merged proposal. + const target = + candidates.find( + (entry) => pr && entry.pr_json && JSON.parse(entry.pr_json).number === pr.number, + ) ?? + candidates.find((entry) => !entry.pr_json) ?? + candidates[0] + if (target) return `https://tips.sh${pagePath(target.number)}${file[2] ?? ''}` + } + if (url.startsWith('/') && !file) return new URL(url, 'https://tips.sh').href + return new URL(url, source).href + } + + visit(tree, (node) => { + if (node.type === 'link' || node.type === 'image' || node.type === 'definition') { + node.url = resolve(node.url) + } + if (node.type === 'html') { + node.value = node.value.replace( + /\b(href|src)=(['"])(.*?)\2/gi, + (_match, attr, quote, url) => `${attr}=${quote}${resolve(url)}${quote}`, + ) + } + }) + return processor.stringify(tree) +} + +export function index(tips: Summary[]) { + return [ + '# Tempo Improvement Proposals', + '', + '> Each link serves the complete TIP as Markdown. Proposal variants are separate entries.', + '', + ...tips.map((tip) => { + const label = `TIP-${tip.number}: ${tip.title}`.replace(/[\\[\]]/g, '\\$&') + const pr = tip.pr_json + ? (JSON.parse(tip.pr_json) as { number: number; url: string }) + : undefined + return `- [${label}](https://tips.sh${pagePath(tip.number)}) (${tip.status})${pr ? ` — [Proposed in PR #${pr.number}](${pr.url})` : ''}` + }), + '', + ].join('\n') +} + +export async function hash(content: string) { + const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(content)) + return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, '0')).join('') +} + +export async function manifest(tips: Tip[]) { + return Promise.all( + tips.map(async (tip) => ({ + id: tip.number, + path: pagePath(tip.number), + sha256: await hash(render(tip, tips)), + })), + ) +} diff --git a/src/lib/MarkdownHttp.test.ts b/src/lib/MarkdownHttp.test.ts new file mode 100644 index 0000000..c123ced --- /dev/null +++ b/src/lib/MarkdownHttp.test.ts @@ -0,0 +1,153 @@ +import { describe, expect, it } from 'vite-plus/test' +import { handle, shouldServeMarkdown, vary } from './MarkdownHttp' +import * as Export from './MarkdownExport' +import type { Row } from './TipStore' + +const rows: Row[] = ['0000', '1034', '1034-1', 'configurable-accounts', 'v2.0-accounts'].map( + (number) => ({ + number, + title: `Proposal ${number}`, + filename: `tip-${number.split('-')[0]}.md`, + content: `---\nid: TIP-${number}\ntitle: Proposal ${number}\n---\n\n# TIP-${number}\n\nComplete specification.\n`, + status: 'Draft', + authors: 'Author', + abstract: '', + protocol_version: '', + created_at: '2026-01-01', + pr_json: number.includes('-') + ? JSON.stringify({ number: 42, url: 'https://github.com/tempoxyz/tempo/pull/42' }) + : '', + }), +) + +function database(inventory = rows) { + return { + prepare() { + return { + bind(id: string) { + return { + async first() { + return inventory.find((tip) => tip.number === id) ?? null + }, + } + }, + async all() { + return { results: inventory } + }, + } + }, + } as unknown as D1Database +} + +function request(path: string, headers: HeadersInit = {}, method = 'GET') { + return new Request(`https://tips.sh${path}`, { method, headers }) +} + +describe('Markdown HTTP', () => { + it('serves every inventory entry explicitly and by negotiation with identical content', async () => { + const manifest = await Export.manifest(rows) + for (const entry of manifest) { + for (const req of [ + request(entry.path, { Accept: 'text/html', 'User-Agent': 'Slackbot' }), + request(`/${entry.id}`, { Accept: 'text/markdown', 'User-Agent': 'Mozilla/5.0' }), + request(`/${entry.id}/`, { 'User-Agent': 'curl/8' }), + ]) { + const response = (await handle(req, async () => database()))! + expect(response.status).toBe(200) + expect(response.headers.get('content-type')).toBe('text/markdown; charset=utf-8') + expect(await Export.hash(await response.text())).toBe(entry.sha256) + } + } + }) + + it('discovers every variant in explicit indexes and the negotiated root', async () => { + for (const path of ['/llms.txt', '/index.md', '/']) { + const response = (await handle(request(path, { Accept: 'text/markdown' }), async () => + database(), + ))! + const body = await response.text() + for (const row of rows) + expect(body).toContain(`https://tips.sh${Export.pagePath(row.number)})`) + expect(body).toContain('[Proposed in PR #42](https://github.com/tempoxyz/tempo/pull/42)') + } + }) + + it('keeps HTML, preview bots, framework POSTs, and assets out of the database', async () => { + const fail = async (): Promise => { + throw new Error('unexpected database read') + } + for (const req of [ + request('/1034', { Accept: 'text/html', 'User-Agent': 'Mozilla/5.0' }), + request('/1034', { Accept: '*/*', 'User-Agent': 'Slackbot' }), + request('/1034', { 'User-Agent': 'Googlebot' }), + request('/1034', { Accept: 'text/markdown' }, 'POST'), + request('/1034.md', {}, 'POST'), + request('/og/1034.png', { Accept: 'text/markdown' }), + ]) + expect(await handle(req, fail)).toBeUndefined() + }) + + it('returns explicit missing and database failure responses instead of HTML', async () => { + for (const path of ['/missing.md', '/missing']) { + const response = (await handle(request(path, { Accept: 'text/markdown' }), async () => + database(), + ))! + expect(response.status).toBe(404) + expect(await response.text()).toBe('TIP not found\n') + } + const response = (await handle(request('/1034.md'), async () => { + throw new Error('database offline') + }))! + expect(response.status).toBe(503) + expect(response.headers.get('cache-control')).toBe('no-store') + expect(await response.text()).toBe('Markdown unavailable\n') + }) + + it('returns matching HEAD headers without a body', async () => { + const get = (await handle(request('/1034-1.md'), async () => database()))! + const head = (await handle(request('/1034-1.md', {}, 'HEAD'), async () => database()))! + expect([...head.headers]).toEqual([...get.headers]) + expect(head.status).toBe(get.status) + expect(await head.text()).toBe('') + }) + + it('varies both representations and prevents shared caching of negotiated URLs', async () => { + const html = vary( + new Response('', { + headers: { Vary: 'Accept-Encoding', 'Cache-Control': 'public' }, + }), + ) + expect(html.headers.get('vary')).toBe('Accept-Encoding, Accept, User-Agent') + expect(html.headers.get('cache-control')).toBe('private, no-store') + const markdown = (await handle(request('/1034', { Accept: 'text/markdown' }), async () => + database(), + ))! + expect(markdown.headers.get('vary')).toBe('Accept, User-Agent') + expect(markdown.headers.get('cache-control')).toBe('private, no-store') + const explicit = (await handle(request('/1034.md'), async () => database()))! + expect(explicit.headers.get('cache-control')).toBe('public, max-age=0, s-maxage=300') + }) +}) + +describe('content negotiation', () => { + it.each([ + ['text/markdown', 'Mozilla/5.0', true], + ['TEXT/MARKDOWN; charset=utf-8', 'Mozilla/5.0', true], + ['text/html;q=0.5, text/markdown;q=0.9', 'Mozilla/5.0', true], + ['text/html, text/markdown;q=0.5', 'curl/8', false], + ['text/markdown;q=0, */*;q=1', 'curl/8', false], + ['text/markdown;q=invalid', 'ClaudeBot', false], + ['text/html', 'ClaudeBot', false], + ['*/*', 'curl/8', true], + ['', 'ClaudeBot', true], + ['*/*;q=0', 'curl/8', false], + ['text/markdown', 'Twitterbot', true], + ['text/markdown', 'Googlebot', true], + ['*/*', 'Twitterbot', false], + ['*/*', 'Mozilla/5.0', false], + ])('Accept %s, UA %s → Markdown %s', (accept, ua, expected) => { + expect(shouldServeMarkdown(request('/1034', { Accept: accept, 'User-Agent': ua }))).toBe( + expected, + ) + }) +}) diff --git a/src/lib/MarkdownHttp.ts b/src/lib/MarkdownHttp.ts new file mode 100644 index 0000000..c9ba7a9 --- /dev/null +++ b/src/lib/MarkdownHttp.ts @@ -0,0 +1,142 @@ +/** HTTP representation selection, shared inventory, and explicit Markdown routes. */ +import * as Export from './MarkdownExport' +import * as TipStore from './TipStore' + +const aiUserAgents = [ + 'GPTBot', + 'OAI-SearchBot', + 'ChatGPT-User', + 'ChatGPT-User/2.0', + 'anthropic-ai', + 'ClaudeBot', + 'claude-web', + 'PerplexityBot', + 'Perplexity-User', + 'Google-Extended', + 'FacebookBot', + 'meta-externalagent', + 'Bytespider', + 'cohere-ai', + 'AI2Bot', + 'CCBot', + 'Diffbot', + 'omgili', + 'Timpibot', + 'MistralAI-User', + 'GoogleAgent-Mariner', +] +const searchEngineUserAgents = [ + 'Googlebot', + 'Bingbot', + 'Amazonbot', + 'Applebot', + 'Applebot-Extended', + 'DuckAssistBot', + 'YouBot', +] +const terminalUserAgents = ['curl/', 'Wget/', 'HTTPie/', 'httpie-go/', 'xh/'] +const ogBotUserAgents = [ + 'Discordbot', + 'Embedly', + 'Facebot', + 'Iframely', + 'LinkedInBot', + 'Pinterestbot', + 'Slackbot', + 'Slurp', + 'TelegramBot', + 'Twitterbot', + 'WhatsApp', + 'facebookexternalhit', +] + +export function shouldServeMarkdown(request: Request): boolean { + const ua = request.headers.get('user-agent') ?? '' + const accept = (request.headers.get('accept') ?? '') + .toLowerCase() + .split(',') + .map((entry) => { + const [type, ...params] = entry.trim().split(';') + const raw = params.map((param) => param.trim()).find((param) => param.startsWith('q=')) + const q = raw ? Number(raw.slice(2)) : 1 + return { type: type.trim(), q: Number.isFinite(q) && q >= 0 && q <= 1 ? q : 0 } + }) + const markdown = accept.find((entry) => entry.type === 'text/markdown') + const html = + accept.find((entry) => entry.type === 'text/html') ?? + accept.find((entry) => entry.type === 'text/*') ?? + accept.find((entry) => entry.type === '*/*') + if (markdown) return markdown.q > 0 && markdown.q >= (html?.q ?? 0) + if ([...ogBotUserAgents, ...searchEngineUserAgents].some((agent) => ua.includes(agent))) + return false + if ( + accept.some((entry) => entry.type === 'text/*' && entry.q === 0) || + (!accept.some((entry) => entry.type === 'text/*') && + accept.some((entry) => entry.type === '*/*' && entry.q === 0)) + ) + return false + if (accept.some((entry) => entry.type === 'text/html' && entry.q > 0)) return false + return [...aiUserAgents, ...terminalUserAgents].some((agent) => ua.includes(agent)) +} + +export function vary(response: Response) { + const headers = new Headers(response.headers) + const values = (headers.get('Vary') ?? '') + .split(',') + .map((value) => value.trim()) + .filter(Boolean) + if (!values.includes('*')) { + for (const name of ['Accept', 'User-Agent']) { + if (!values.some((value) => value.toLowerCase() === name.toLowerCase())) values.push(name) + } + headers.set('Vary', values.join(', ')) + } + // Cloudflare's default cache key ignores Vary for these headers. Never let + // a negotiated response poison an HTML URL, even when it carries Vary. + headers.set('Cache-Control', 'private, no-store') + return new Response(response.body, { + status: response.status, + statusText: response.statusText, + headers, + }) +} + +export async function handle( + request: Request, + database: () => Promise, +): Promise { + if (request.method !== 'GET' && request.method !== 'HEAD') return + const pathname = new URL(request.url).pathname + const explicitIndex = pathname === '/index.md' || pathname === '/llms.txt' + const explicitTip = pathname.match(/^\/([^/]+)\.md$/) + const negotiated = shouldServeMarkdown(request) + if (!explicitIndex && !explicitTip && !negotiated) return + const tipMatch = pathname.match(/^\/([^/]+)\/?$/) + if (!explicitIndex && !explicitTip && pathname !== '/' && !tipMatch) return + const response = (body: string, status = 200) => + new Response(request.method === 'HEAD' ? null : body, { + status, + headers: { + 'Content-Type': 'text/markdown; charset=utf-8', + 'Cache-Control': + explicitIndex || explicitTip ? 'public, max-age=0, s-maxage=300' : 'private, no-store', + Vary: 'Accept, User-Agent', + }, + }) + try { + const db = await database() + if (explicitIndex || pathname === '/') return response(Export.index(await TipStore.list(db))) + const id = decodeURIComponent((explicitTip ?? tipMatch)![1]) + const tip = await TipStore.get(db, id) + if (!tip) { + if (!explicitTip && id.includes('.')) return + return response('TIP not found\n', 404) + } + return response(Export.render(tip, await TipStore.list(db))) + } catch (error) { + console.error('[md-router]', error) + const failed = response('Markdown unavailable\n', 503) + failed.headers.set('Cache-Control', 'no-store') + return failed + } +} diff --git a/src/lib/MdRouter.ts b/src/lib/MdRouter.ts index fdd283b..b6576b4 100644 --- a/src/lib/MdRouter.ts +++ b/src/lib/MdRouter.ts @@ -2,65 +2,7 @@ import { createMiddleware } from '@tanstack/react-start' import * as Markdown from '#/lib/Markdown' - -const aiUserAgents = [ - 'GPTBot', - 'OAI-SearchBot', - 'ChatGPT-User', - 'ChatGPT-User/2.0', - 'anthropic-ai', - 'ClaudeBot', - 'claude-web', - 'PerplexityBot', - 'Perplexity-User', - 'Google-Extended', - 'FacebookBot', - 'meta-externalagent', - 'Bytespider', - 'cohere-ai', - 'AI2Bot', - 'CCBot', - 'Diffbot', - 'omgili', - 'Timpibot', - 'MistralAI-User', - 'GoogleAgent-Mariner', -] -const searchEngineUserAgents = [ - 'Googlebot', - 'Bingbot', - 'Amazonbot', - 'Applebot', - 'Applebot-Extended', - 'DuckAssistBot', - 'YouBot', -] -const terminalUserAgents = ['curl/', 'Wget/', 'HTTPie/', 'httpie-go/', 'xh/'] -const ogBotUserAgents = [ - 'Discordbot', - 'Embedly', - 'Facebot', - 'Iframely', - 'LinkedInBot', - 'Pinterestbot', - 'Slackbot', - 'Slurp', - 'TelegramBot', - 'Twitterbot', - 'WhatsApp', - 'facebookexternalhit', -] - -function shouldServeMarkdown(request: Request): boolean { - const ua = request.headers.get('user-agent') ?? '' - const accept = request.headers.get('accept') ?? '' - if (ogBotUserAgents.some((a) => ua.includes(a))) return false - if (searchEngineUserAgents.some((a) => ua.includes(a))) return false - if (aiUserAgents.some((a) => ua.includes(a))) return true - if (terminalUserAgents.some((a) => ua.includes(a))) return true - if (accept.includes('text/markdown')) return true - return false -} +import * as MarkdownHttp from '#/lib/MarkdownHttp' export const middleware = createMiddleware({ type: 'request' }).server( async ({ request, next }) => { @@ -69,7 +11,7 @@ export const middleware = createMiddleware({ type: 'request' }).server( const tipFile = pathname.match(/^\/tip-(\d+)\.md$/) if (tipFile) { - url.pathname = `/${tipFile[1]}` + url.pathname = `/${tipFile[1]}.md` throw Response.redirect(url.toString(), 308) } @@ -123,62 +65,11 @@ export const middleware = createMiddleware({ type: 'request' }).server( } } - // /:id.md → always serve markdown regardless of user-agent - const mdMatch = pathname.match(/^\/(\d+)\.md$/) - if (mdMatch) { - try { - const { env } = await import('cloudflare:workers') - const tip = await env.DB.prepare('SELECT content FROM tips WHERE number = ?') - .bind(mdMatch[1]) - .first<{ content: string }>() - if (tip) - throw new Response(tip.content, { - headers: { 'Content-Type': 'text/markdown; charset=utf-8' }, - }) - } catch (e) { - if (e instanceof Response) throw e - console.error('[md-router]', e) - } - return next() - } - - if (!shouldServeMarkdown(request)) return next() - - try { + const markdown = await MarkdownHttp.handle(request, async () => { const { env } = await import('cloudflare:workers') - const db = env.DB - - const tipMatch = pathname.match(/^\/(\d+)$/) - if (tipMatch) { - const tip = await db - .prepare('SELECT content FROM tips WHERE number = ?') - .bind(tipMatch[1]) - .first<{ content: string }>() - if (tip) - throw new Response(tip.content, { - headers: { 'Content-Type': 'text/markdown; charset=utf-8' }, - }) - } - - if (pathname === '/') { - const rows = await db - .prepare('SELECT number, title, status FROM tips ORDER BY CAST(number AS INTEGER)') - .all<{ number: string; title: string; status: string }>() - if (rows.results.length > 0) { - const lines = [ - '# Tempo Improvement Proposals\n', - ...rows.results.map((t) => `- **TIP-${t.number}**: ${t.title} (${t.status})`), - ] - throw new Response(lines.join('\n'), { - headers: { 'Content-Type': 'text/markdown; charset=utf-8' }, - }) - } - } - } catch (e) { - if (e instanceof Response) throw e - console.error('[md-router]', e) - } - + return env.DB + }) + if (markdown) throw markdown return next() }, ) diff --git a/src/lib/TipStore.ts b/src/lib/TipStore.ts new file mode 100644 index 0000000..b8d0938 --- /dev/null +++ b/src/lib/TipStore.ts @@ -0,0 +1,28 @@ +/** The content inventory shared by HTML pages and Markdown exports. */ +export type Row = { + number: string + title: string + authors: string + status: string + abstract: string + content: string + filename: string + protocol_version: string + pr_json: string + created_at: string +} + +export type Summary = Pick + +export function get(db: D1Database, id: string) { + return db.prepare('SELECT * FROM tips WHERE number = ?').bind(id).first() +} + +export async function list(db: D1Database) { + const rows = await db + .prepare( + `SELECT number, title, status, filename, pr_json FROM tips ORDER BY CASE WHEN number GLOB '[0-9]*' THEN 0 ELSE 1 END, CAST(number AS INTEGER), number`, + ) + .all() + return rows.results +} diff --git a/src/lib/Tips.fns.ts b/src/lib/Tips.fns.ts index 585ffaf..fb89fd4 100644 --- a/src/lib/Tips.fns.ts +++ b/src/lib/Tips.fns.ts @@ -4,6 +4,7 @@ import { createServerFn } from '@tanstack/react-start' import { z } from 'zod' import type { Summary, Detail } from './Tips' import * as Markdown from './Markdown' +import * as TipStore from './TipStore' async function db() { const { env } = await import('cloudflare:workers') @@ -23,20 +24,8 @@ async function sha256(content: string): Promise { /** List all TIPs ordered by number. */ export const list = createServerFn().handler(async () => { - const rows = await ( - await db() - ) - .prepare( - `SELECT number, title, status, filename, pr_json FROM tips ORDER BY CASE WHEN number GLOB '[0-9]*' THEN 0 ELSE 1 END, CAST(number AS INTEGER)`, - ) - .all<{ - number: string - title: string - status: string - filename: string - pr_json: string - }>() - return rows.results.map( + const rows = await TipStore.list(await db()) + return rows.map( (r) => ({ number: r.number, @@ -52,25 +41,7 @@ export const list = createServerFn().handler(async () => { export const get = createServerFn({ method: 'POST' }) .inputValidator(z.string()) .handler(async ({ data: tipId }) => { - const row = await ( - await db() - ) - .prepare( - 'SELECT number, title, authors, status, abstract, content, filename, protocol_version, pr_json, created_at FROM tips WHERE number = ?', - ) - .bind(tipId) - .first<{ - number: string - title: string - authors: string - status: string - abstract: string - content: string - filename: string - protocol_version: string - pr_json: string - created_at: string - }>() + const row = await TipStore.get(await db(), tipId) if (!row) throw new Error(`TIP ${tipId} not found`) const contentHash = await sha256(row.content) diff --git a/src/worker.tsx b/src/worker.tsx index cfc7a2e..f2c1cfb 100644 --- a/src/worker.tsx +++ b/src/worker.tsx @@ -2,6 +2,7 @@ import { createStartHandler, defaultStreamHandler } from '@tanstack/react-start/ import { ImageResponse } from 'takumi-js/response' import wasmModule, { initSync, Renderer } from 'takumi-js/wasm' import * as Config from './lib/Config' +import { vary } from './lib/MarkdownHttp' import { OgCard, OgIndex } from './lib/Og' // @ts-expect-error bytes import import cmunrmData from '../public/fonts/cmunrm-clean.ttf?bytes' @@ -64,7 +65,9 @@ export default { } if (url.pathname === '/sitemap.xml' && request.method === 'GET') { - const rows = await env.DB.prepare("SELECT number FROM tips WHERE pr_json = '' OR pr_json IS NULL").all<{ + const rows = await env.DB.prepare( + "SELECT number FROM tips WHERE pr_json = '' OR pr_json IS NULL", + ).all<{ number: string }>() const urls = rows.results @@ -84,10 +87,20 @@ export default { } const response = await handler(request) - if (request.method === 'GET' && response.status === 200) { + if ((request.method === 'GET' || request.method === 'HEAD') && response.status === 200) { const headers = new Headers(response.headers) - headers.set('Cache-Control', 'public, max-age=0, s-maxage=300, stale-while-revalidate=86400') - return new Response(response.body, { status: response.status, headers }) + if (!headers.has('Cache-Control')) + headers.set( + 'Cache-Control', + 'public, max-age=0, s-maxage=300, stale-while-revalidate=86400', + ) + const result = new Response(response.body, { status: response.status, headers }) + const type = headers.get('Content-Type') ?? '' + const negotiated = + !url.pathname.endsWith('.md') && + url.pathname !== '/llms.txt' && + (type.includes('text/html') || type.includes('text/markdown')) + return negotiated ? vary(result) : result } return response }, diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 0000000..fcab011 --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,4 @@ +// Unit tests exercise the content pipeline without booting a Cloudflare Worker. +import { defineConfig } from 'vite-plus' + +export default defineConfig({ test: { environment: 'node' } }) From ee8fc1c4b952fc0eb8ed5afd47ccd08349876783 Mon Sep 17 00:00:00 2001 From: Brendan Ryan <1572504+brendanjryan@users.noreply.github.com> Date: Sun, 18 Oct 2026 17:11:58 +0000 Subject: [PATCH 2/2] chore: remove README changes Co-authored-by: Derek Cofausper <256792747+decofe@users.noreply.github.com> --- README.md | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/README.md b/README.md index e143a13..8315dc9 100644 --- a/README.md +++ b/README.md @@ -1,32 +1 @@ # tips.sh - -## Markdown - -Every TIP page has a `/.md` endpoint, including proposal variants such as -`/1061-1.md` and title-based slugs. These URLs always return Markdown, regardless -of user agent. `GET /` with `Accept: text/markdown` returns the same content; -normal browser requests and link-preview bots retain HTML. Negotiated URLs use -`Vary: Accept, User-Agent` and `Cache-Control: private, no-store` because the -Cloudflare default cache key does not distinguish those headers. Explicit `.md` -URLs can be cached independently. Unknown Markdown IDs return 404, and database -failures return 503 rather than falling back to HTML. - -`/llms.txt` and `/index.md` list the complete website inventory with links to each -Markdown page and the originating PR for proposals. Exports preserve frontmatter, -code, tables, math, and Mermaid source. Relative TIP links resolve to the matching -proposal revision where available; other relative assets resolve against the -source repository or PR. - -Run `pnpm test` for routing, negotiation, content preservation, and inventory -coverage checks. `pnpm sync` also writes `.artifacts/markdown-manifest.json`, -containing expected hashes from that sync. After deploying, verify the public -endpoints against that local file: - -```sh -pnpm check:markdown --url https://tips.sh --manifest .artifacts/markdown-manifest.json -``` - -The deployment workflow runs this check after publishing. Generate the expected -manifest from the intended sync; do not substitute a manifest downloaded from -the deployed site. A concurrent content sync can require rerunning the sync and -verification together.