feat: serve the site in a form agents can read

.gitignore

@@ -30,3 +30,7 @@ next-env.d.ts
 
 # pagefind search index (generated at build)
 /public/_pagefind
+
+# agent files (generated at build from content/)
+/public/*.md
+/public/llms-full.txt

app/[[...mdxPath]]/page.jsx

@@ -1,12 +1,23 @@
 import { generateStaticParamsFor, importPage } from 'nextra/pages'
 import { useMDXComponents as getMDXComponents } from '../../mdx-components.js'
+import { structuredData } from '../structured-data.js'
 
 export const generateStaticParams = generateStaticParamsFor('mdxPath')
 
+// The page's name in content/: `index` for the home page.
+function pageName(params) {
+  return params.mdxPath?.length ? params.mdxPath.join('/') : 'index'
+}
+
 export async function generateMetadata(props) {
   const params = await props.params
   const { metadata } = await importPage(params.mdxPath)
-  return metadata
+  // Every page has a plain Markdown twin, generated at build from the same
+  // source, for agents and answer engines that would rather not parse HTML.
+  return {
+    ...metadata,
+    alternates: { types: { 'text/markdown': `/${pageName(params)}.md` } },
+  }
 }
 
 const Wrapper = getMDXComponents().wrapper
@@ -15,8 +26,12 @@ export default async function Page(props) {
   const params = await props.params
   const result = await importPage(params.mdxPath)
   const { default: MDXContent, toc, metadata } = result
+  const data = structuredData(pageName(params))
   return (
     <Wrapper toc={toc} metadata={metadata}>
+      {data && (
+        <script type="application/ld+json" dangerouslySetInnerHTML={{ __html: JSON.stringify(data) }} />
+      )}
       <MDXContent {...props} params={params} />
     </Wrapper>
   )

app/layout.jsx

@@ -3,16 +3,7 @@ import { Head } from 'nextra/components'
 import { getPageMap } from 'nextra/page-map'
 import 'nextra-theme-docs/style.css'
 import './globals.css'
-
-// Base URL for absolute OG/canonical links. Sourced from the environment so the
-// production domain lives in the Vercel dashboard, never committed here. On Vercel
-// VERCEL_PROJECT_PRODUCTION_URL is injected automatically; set NEXT_PUBLIC_SITE_URL
-// to pin the canonical custom domain.
-const siteUrl =
-  process.env.NEXT_PUBLIC_SITE_URL ||
-  (process.env.VERCEL_PROJECT_PRODUCTION_URL
-    ? `https://${process.env.VERCEL_PROJECT_PRODUCTION_URL}`
-    : 'http://localhost:3000')
+import { siteUrl } from './site-url.js'
 
 const description =
   'A native Nostr ecosystem in Zig: a foundational protocol library, utility libraries, and fast native apps like the Notary remote signer.'

app/robots.js

@@ -0,0 +1,10 @@
+import { siteUrl } from './site-url.js'
+
+// Everyone may read everything, search engines and answer engines alike. The
+// sitemap line is what makes this file worth serving.
+export default function robots() {
+  return {
+    rules: [{ userAgent: '*', allow: '/' }],
+    sitemap: `${siteUrl}/sitemap.xml`,
+  }
+}

app/site-url.js

@@ -0,0 +1,10 @@
+// Base URL for absolute OG, canonical, sitemap and structured-data links.
+// Sourced from the environment so the production domain lives in the Vercel
+// dashboard, never committed here. On Vercel VERCEL_PROJECT_PRODUCTION_URL is
+// injected automatically; set NEXT_PUBLIC_SITE_URL to pin the canonical custom
+// domain.
+export const siteUrl =
+  process.env.NEXT_PUBLIC_SITE_URL ||
+  (process.env.VERCEL_PROJECT_PRODUCTION_URL
+    ? `https://${process.env.VERCEL_PROJECT_PRODUCTION_URL}`
+    : 'http://localhost:3000')

app/sitemap.js

@@ -0,0 +1,16 @@
+import { readdirSync } from 'node:fs'
+import { join } from 'node:path'
+import { siteUrl } from './site-url.js'
+
+// One entry per page in content/, so a new page is in the sitemap the moment
+// it exists.
+export default function sitemap() {
+  const pages = readdirSync(join(process.cwd(), 'content'))
+    .filter((f) => f.endsWith('.mdx'))
+    .map((f) => f.slice(0, -4))
+  return pages.map((p) => ({
+    url: p === 'index' ? `${siteUrl}/` : `${siteUrl}/${p}`,
+    changeFrequency: 'weekly',
+    priority: p === 'index' ? 1 : 0.7,
+  }))
+}

app/structured-data.js

@@ -0,0 +1,58 @@
+import { siteUrl } from './site-url.js'
+
+// schema.org descriptions of what each page is about, as JSON-LD, so search
+// and answer engines can tell a library from an app from a command line.
+
+const org = {
+  '@type': 'Organization',
+  '@id': `${siteUrl}/#org`,
+  name: 'zig-nostr',
+  url: `${siteUrl}/`,
+  logo: `${siteUrl}/logo.svg`,
+  sameAs: ['https://github.com/zig-nostr'],
+}
+
+const app = (name, page, repo, description, extra = {}) => ({
+  '@type': 'SoftwareApplication',
+  name,
+  url: `${siteUrl}/${page}`,
+  description,
+  applicationCategory: 'DeveloperApplication',
+  operatingSystem: 'macOS, Linux',
+  license: 'https://opensource.org/licenses/MIT',
+  offers: { '@type': 'Offer', price: '0', priceCurrency: 'USD' },
+  codeRepository: `https://github.com/zig-nostr/${repo}`,
+  downloadUrl: `https://github.com/zig-nostr/${repo}/releases/latest`,
+  publisher: { '@id': `${siteUrl}/#org` },
+  ...extra,
+})
+
+const library = {
+  '@type': 'SoftwareSourceCode',
+  name: 'nostr',
+  alternateName: 'zig-nostr',
+  description: 'A Nostr protocol library for Zig: keys and BIP-340 signatures, NIP-01 events, NIP-19, NIP-44, NIP-46, relay transport, the NIP-65 outbox model, and a local-first LMDB event store.',
+  codeRepository: 'https://github.com/zig-nostr/nostr',
+  programmingLanguage: 'Zig',
+  license: 'https://opensource.org/licenses/MIT',
+  url: `${siteUrl}/getting-started`,
+  publisher: { '@id': `${siteUrl}/#org` },
+}
+
+const byPage = {
+  index: [
+    org,
+    { '@type': 'WebSite', '@id': `${siteUrl}/#site`, name: 'zig-nostr', url: `${siteUrl}/`, publisher: { '@id': `${siteUrl}/#org` } },
+    library,
+  ],
+  'getting-started': [library],
+  deed: [app('deed', 'deed', 'deed', 'The nostr command line: keys, events, NIP-19, NIP-44, relay queries, publishing, and a local store that later runs read without a network.')],
+  plaza: [app('Plaza', 'plaza', 'plaza', 'A fast, local-first Nostr client for macOS and Linux, rendered from its own store and reconciled with relays in the background.', { applicationCategory: 'SocialNetworkingApplication' })],
+  notary: [app('Notary', 'notary', 'notary', 'A native NIP-46 remote signer for macOS and Linux: the key stays in a local daemon on a machine you control.', { applicationCategory: 'SecurityApplication' })],
+}
+
+export function structuredData(page) {
+  const items = byPage[page]
+  if (!items) return null
+  return { '@context': 'https://schema.org', '@graph': items }
+}

package.json

@@ -5,6 +5,7 @@
   "type": "module",
   "scripts": {
     "dev": "next dev",
+    "prebuild": "node scripts/agent-files.mjs",
     "build": "next build",
     "start": "next start",
     "postbuild": "pagefind --site .next/server/app/ --output-path public/_pagefind"

public/07d125e7231655dd8670b5fb9f6f9aed.txt

@@ -0,0 +1 @@
+07d125e7231655dd8670b5fb9f6f9aed

public/llms.txt

@@ -0,0 +1,31 @@
+# zig-nostr
+
+> A native Nostr ecosystem in Zig: `nostr`, a protocol library for Zig; Plaza, a local-first client for macOS and Linux; Notary, a NIP-46 remote signer; and deed, a command line for nostr. MIT licensed, all on one core.
+
+Every page on this site is also served as Markdown at its path with `.md` added (the home page is /index.md), and /llms-full.txt has them all in one file.
+
+## Projects
+
+- [nostr, the Zig library](/getting-started.md): keys and BIP-340 signatures via libsecp256k1, NIP-01 events, NIP-19, NIP-44, NIP-46, NIP-42, NIP-49, NIP-06, relay transport with deadlines, the NIP-65 outbox model, and a memory-mapped LMDB event store. Source: https://github.com/zig-nostr/nostr
+- [deed, the nostr command line](/deed.md): keys, signing, verifying, NIP-19, NIP-44, relay queries, publishing, and a local store it can query offline. One binary for macOS and Linux. Source: https://github.com/zig-nostr/deed
+- [Plaza](/plaza.md): a fast, local-first nostr client for macOS and Linux, rendered from its own store. Source: https://github.com/zig-nostr/plaza
+- [Notary](/notary.md): a native NIP-46 remote signer for macOS and Linux; the key stays in a local daemon on a machine you control. Source: https://github.com/zig-nostr/notary
+
+## For agents
+
+- [Skill: using deed](https://github.com/zig-nostr/deed/blob/main/skills/deed/SKILL.md): install with `npx skills add zig-nostr/deed`. Covers every command, recipes, exit codes, and two rules: publishing is public and permanent, so ask the user first, and keep secret keys in `NOSTR_SECRET_KEY`.
+- [Skill: building with the Zig library](https://github.com/zig-nostr/nostr/blob/main/skills/zig-nostr/SKILL.md): install with `npx skills add zig-nostr/nostr`. Adding the dependency, keys, events, NIP-19, NIP-44, relays read with a deadline, the store, and the traps.
+
+## Docs
+
+- [Getting started](/getting-started.md): add the library and sign a first event
+- [Ecosystem](/ecosystem.md): what exists and how the pieces fit
+- [Architecture](/architecture.md): how the library is put together
+- [NIP support](/nips.md): which NIPs are implemented, and where
+- [Performance](/performance.md): store benchmarks for the library, and deed's numbers
+- [Roadmap](/roadmap.md): what comes next
+
+## Optional
+
+- [Everything in one file](/llms-full.txt)
+- [deed benchmarks](https://github.com/zig-nostr/deed/blob/main/BENCHMARKS.md)

scripts/agent-files.mjs

@@ -0,0 +1,61 @@
+// Writes the files agents and answer engines read, from the same MDX the site
+// renders, so they can never drift from the pages:
+//
+//   public/<page>.md     a plain Markdown copy of every page (index.md for home)
+//   public/llms-full.txt every page in one file, in navigation order
+//
+// Runs before `next build`. The output is generated, so it is ignored by git.
+import { readFileSync, writeFileSync, readdirSync } from 'node:fs'
+import { join } from 'node:path'
+
+const root = new URL('..', import.meta.url).pathname
+const content = join(root, 'content')
+const out = join(root, 'public')
+
+// Navigation order comes from _meta.js, so llms-full.txt reads like the site.
+const meta = readFileSync(join(content, '_meta.js'), 'utf8')
+const order = [...meta.matchAll(/^\s{2}(?:'([^']+)'|(\w+)):/gm)].map((m) => m[1] || m[2])
+const pages = readdirSync(content).filter((f) => f.endsWith('.mdx')).map((f) => f.slice(0, -4))
+const ordered = [...order.filter((p) => pages.includes(p)), ...pages.filter((p) => !order.includes(p))]
+
+const ELEMENT = '(?:div|span|p|img|h[1-6]|a|br|section|ul|ol|li|strong|em|b|i|code|pre|small)'
+
+function toMarkdown(src) {
+  let title = null
+  src = src.replace(/^---\n([\s\S]*?)\n---\n/, (_, fm) => {
+    const t = fm.match(/^title:\s*(.+)$/m)
+    if (t) title = t[1].trim()
+    return ''
+  })
+  // Protect code, fenced and inline, from the tag stripping below.
+  const kept = []
+  const keep = (s) => `\u0000${kept.push(s) - 1}\u0000`
+  src = src.replace(/```[\s\S]*?```/g, keep).replace(/`[^`\n]+`/g, keep)
+
+  src = src
+    .replace(/^(?:import|export) .*$/gm, '')
+    .replace(/<h([1-6])[^>]*>([\s\S]*?)<\/h\1>/g, (_, n, t) => `\n\n${'#'.repeat(+n)} ${t.trim()}\n\n`)
+    .replace(/<a\b[^>]*?href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/g, (_, href, t) => `[${t.trim()}](${href})`)
+    .replace(/<p\b[^>]*>([\s\S]*?)<\/p>/g, (_, t) => `\n\n${t.trim().replace(/\s*\n\s*/g, ' ')}\n\n`)
+    .replace(new RegExp(`</?${ELEMENT}\\b[^>]*?/?>`, 'g'), '\n')
+    .replace(/&nbsp;/g, ' ')
+    .replace(/&amp;/g, '&')
+    .replace(/[ \t]+$/gm, '')
+    // Indentation survives only on nested list items; anywhere else four
+    // spaces would turn a line into a code block.
+    .replace(/^[ \t]+(?=\S)/gm, (ws, off, all) => (/^[ \t]+(?:[-*+]|\d+\.)\s/.test(all.slice(off, off + ws.length + 4)) ? ws : ''))
+    .replace(/\n{3,}/g, '\n\n')
+    .trim()
+  src = src.replace(/\u0000(\d+)\u0000/g, (_, i) => kept[+i])
+  if (title && !/^# /.test(src)) src = `# ${title}\n\n${src}`
+  return src + '\n'
+}
+
+const full = ['# zig-nostr, every page\n', 'The whole site as Markdown, in navigation order. Each page is also served on its own at its path with `.md` added; the home page is `/index.md`.\n']
+for (const page of ordered) {
+  const md = toMarkdown(readFileSync(join(content, `${page}.mdx`), 'utf8'))
+  writeFileSync(join(out, `${page}.md`), md)
+  full.push(`\n---\n\nSource: /${page === 'index' ? '' : page}\n\n${md}`)
+}
+writeFileSync(join(out, 'llms-full.txt'), full.join(''))
+console.log(`agent files: ${ordered.length} pages, llms-full.txt`)