From b7a3ecd1c25d51cc037c1713a0e1ad33b3a69021 Mon Sep 17 00:00:00 2001
From: sepehr-safari <safari.sepehr@gmail.com>
Date: Thu, 24 Sep 2026 12:08:24 +0300
Subject: [PATCH] feat: serve the site in a form agents can read

Every page now has a Markdown twin at its path with .md added, generated at
build from the same MDX and linked from the page as a text/markdown
alternate. llms.txt indexes the projects and docs, llms-full.txt carries every
page in one file, and sitemap.xml and robots.txt are built from content/ so a
new page is listed the moment it exists.

Each page also carries schema.org JSON-LD saying what it describes: the
library as source code, Plaza, Notary and deed as applications, the home page
as the organization and the site. The IndexNow key file lets changed pages be
announced to search engines directly.
---
 .gitignore                                  |  4 ++
 app/[[...mdxPath]]/page.jsx                 | 17 +++++-
 app/layout.jsx                              | 11 +---
 app/robots.js                               | 10 ++++
 app/site-url.js                             | 10 ++++
 app/sitemap.js                              | 16 ++++++
 app/structured-data.js                      | 58 ++++++++++++++++++++
 package.json                                |  1 +
 public/07d125e7231655dd8670b5fb9f6f9aed.txt |  1 +
 public/llms.txt                             | 31 +++++++++++
 scripts/agent-files.mjs                     | 61 +++++++++++++++++++++
 11 files changed, 209 insertions(+), 11 deletions(-)
 create mode 100644 app/robots.js
 create mode 100644 app/site-url.js
 create mode 100644 app/sitemap.js
 create mode 100644 app/structured-data.js
 create mode 100644 public/07d125e7231655dd8670b5fb9f6f9aed.txt
 create mode 100644 public/llms.txt
 create mode 100644 scripts/agent-files.mjs

diff --git a/.gitignore b/.gitignore
index a3ef750..2ebc802 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,3 +30,7 @@ next-env.d.ts
 
 # pagefind search index (generated at build)
 /public/_pagefind
+
+# agent files (generated at build from content/)
+/public/*.md
+/public/llms-full.txt
diff --git a/app/[[...mdxPath]]/page.jsx b/app/[[...mdxPath]]/page.jsx
index da021c8..e63c35c 100644
--- a/app/[[...mdxPath]]/page.jsx
+++ b/app/[[...mdxPath]]/page.jsx
@@ -1,12 +1,23 @@
 import { generateStaticParamsFor, importPage } from 'nextra/pages'
 import { useMDXComponents as getMDXComponents } from '../../mdx-components.js'
+import { structuredData } from '../structured-data.js'
 
 export const generateStaticParams = generateStaticParamsFor('mdxPath')
 
+// The page's name in content/: `index` for the home page.
+function pageName(params) {
+  return params.mdxPath?.length ? params.mdxPath.join('/') : 'index'
+}
+
 export async function generateMetadata(props) {
   const params = await props.params
   const { metadata } = await importPage(params.mdxPath)
-  return metadata
+  // Every page has a plain Markdown twin, generated at build from the same
+  // source, for agents and answer engines that would rather not parse HTML.
+  return {
+    ...metadata,
+    alternates: { types: { 'text/markdown': `/${pageName(params)}.md` } },
+  }
 }
 
 const Wrapper = getMDXComponents().wrapper
@@ -15,8 +26,12 @@ export default async function Page(props) {
   const params = await props.params
   const result = await importPage(params.mdxPath)
   const { default: MDXContent, toc, metadata } = result
+  const data = structuredData(pageName(params))
   return (
     <Wrapper toc={toc} metadata={metadata}>
+      {data && (
+        <script type="application/ld+json" dangerouslySetInnerHTML={{ __html: JSON.stringify(data) }} />
+      )}
       <MDXContent {...props} params={params} />
     </Wrapper>
   )
diff --git a/app/layout.jsx b/app/layout.jsx
index 8677303..b152757 100644
--- a/app/layout.jsx
+++ b/app/layout.jsx
@@ -3,16 +3,7 @@ import { Head } from 'nextra/components'
 import { getPageMap } from 'nextra/page-map'
 import 'nextra-theme-docs/style.css'
 import './globals.css'
-
-// Base URL for absolute OG/canonical links. Sourced from the environment so the
-// production domain lives in the Vercel dashboard, never committed here. On Vercel
-// VERCEL_PROJECT_PRODUCTION_URL is injected automatically; set NEXT_PUBLIC_SITE_URL
-// to pin the canonical custom domain.
-const siteUrl =
-  process.env.NEXT_PUBLIC_SITE_URL ||
-  (process.env.VERCEL_PROJECT_PRODUCTION_URL
-    ? `https://${process.env.VERCEL_PROJECT_PRODUCTION_URL}`
-    : 'http://localhost:3000')
+import { siteUrl } from './site-url.js'
 
 const description =
   'A native Nostr ecosystem in Zig: a foundational protocol library, utility libraries, and fast native apps like the Notary remote signer.'
diff --git a/app/robots.js b/app/robots.js
new file mode 100644
index 0000000..9435063
--- /dev/null
+++ b/app/robots.js
@@ -0,0 +1,10 @@
+import { siteUrl } from './site-url.js'
+
+// Everyone may read everything, search engines and answer engines alike. The
+// sitemap line is what makes this file worth serving.
+export default function robots() {
+  return {
+    rules: [{ userAgent: '*', allow: '/' }],
+    sitemap: `${siteUrl}/sitemap.xml`,
+  }
+}
diff --git a/app/site-url.js b/app/site-url.js
new file mode 100644
index 0000000..e01851e
--- /dev/null
+++ b/app/site-url.js
@@ -0,0 +1,10 @@
+// Base URL for absolute OG, canonical, sitemap and structured-data links.
+// Sourced from the environment so the production domain lives in the Vercel
+// dashboard, never committed here. On Vercel VERCEL_PROJECT_PRODUCTION_URL is
+// injected automatically; set NEXT_PUBLIC_SITE_URL to pin the canonical custom
+// domain.
+export const siteUrl =
+  process.env.NEXT_PUBLIC_SITE_URL ||
+  (process.env.VERCEL_PROJECT_PRODUCTION_URL
+    ? `https://${process.env.VERCEL_PROJECT_PRODUCTION_URL}`
+    : 'http://localhost:3000')
diff --git a/app/sitemap.js b/app/sitemap.js
new file mode 100644
index 0000000..6353127
--- /dev/null
+++ b/app/sitemap.js
@@ -0,0 +1,16 @@
+import { readdirSync } from 'node:fs'
+import { join } from 'node:path'
+import { siteUrl } from './site-url.js'
+
+// One entry per page in content/, so a new page is in the sitemap the moment
+// it exists.
+export default function sitemap() {
+  const pages = readdirSync(join(process.cwd(), 'content'))
+    .filter((f) => f.endsWith('.mdx'))
+    .map((f) => f.slice(0, -4))
+  return pages.map((p) => ({
+    url: p === 'index' ? `${siteUrl}/` : `${siteUrl}/${p}`,
+    changeFrequency: 'weekly',
+    priority: p === 'index' ? 1 : 0.7,
+  }))
+}
diff --git a/app/structured-data.js b/app/structured-data.js
new file mode 100644
index 0000000..0772ebd
--- /dev/null
+++ b/app/structured-data.js
@@ -0,0 +1,58 @@
+import { siteUrl } from './site-url.js'
+
+// schema.org descriptions of what each page is about, as JSON-LD, so search
+// and answer engines can tell a library from an app from a command line.
+
+const org = {
+  '@type': 'Organization',
+  '@id': `${siteUrl}/#org`,
+  name: 'zig-nostr',
+  url: `${siteUrl}/`,
+  logo: `${siteUrl}/logo.svg`,
+  sameAs: ['https://github.com/zig-nostr'],
+}
+
+const app = (name, page, repo, description, extra = {}) => ({
+  '@type': 'SoftwareApplication',
+  name,
+  url: `${siteUrl}/${page}`,
+  description,
+  applicationCategory: 'DeveloperApplication',
+  operatingSystem: 'macOS, Linux',
+  license: 'https://opensource.org/licenses/MIT',
+  offers: { '@type': 'Offer', price: '0', priceCurrency: 'USD' },
+  codeRepository: `https://github.com/zig-nostr/${repo}`,
+  downloadUrl: `https://github.com/zig-nostr/${repo}/releases/latest`,
+  publisher: { '@id': `${siteUrl}/#org` },
+  ...extra,
+})
+
+const library = {
+  '@type': 'SoftwareSourceCode',
+  name: 'nostr',
+  alternateName: 'zig-nostr',
+  description: 'A Nostr protocol library for Zig: keys and BIP-340 signatures, NIP-01 events, NIP-19, NIP-44, NIP-46, relay transport, the NIP-65 outbox model, and a local-first LMDB event store.',
+  codeRepository: 'https://github.com/zig-nostr/nostr',
+  programmingLanguage: 'Zig',
+  license: 'https://opensource.org/licenses/MIT',
+  url: `${siteUrl}/getting-started`,
+  publisher: { '@id': `${siteUrl}/#org` },
+}
+
+const byPage = {
+  index: [
+    org,
+    { '@type': 'WebSite', '@id': `${siteUrl}/#site`, name: 'zig-nostr', url: `${siteUrl}/`, publisher: { '@id': `${siteUrl}/#org` } },
+    library,
+  ],
+  'getting-started': [library],
+  deed: [app('deed', 'deed', 'deed', 'The nostr command line: keys, events, NIP-19, NIP-44, relay queries, publishing, and a local store that later runs read without a network.')],
+  plaza: [app('Plaza', 'plaza', 'plaza', 'A fast, local-first Nostr client for macOS and Linux, rendered from its own store and reconciled with relays in the background.', { applicationCategory: 'SocialNetworkingApplication' })],
+  notary: [app('Notary', 'notary', 'notary', 'A native NIP-46 remote signer for macOS and Linux: the key stays in a local daemon on a machine you control.', { applicationCategory: 'SecurityApplication' })],
+}
+
+export function structuredData(page) {
+  const items = byPage[page]
+  if (!items) return null
+  return { '@context': 'https://schema.org', '@graph': items }
+}
diff --git a/package.json b/package.json
index d329b1e..62c6d8f 100644
--- a/package.json
+++ b/package.json
@@ -5,6 +5,7 @@
   "type": "module",
   "scripts": {
     "dev": "next dev",
+    "prebuild": "node scripts/agent-files.mjs",
     "build": "next build",
     "start": "next start",
     "postbuild": "pagefind --site .next/server/app/ --output-path public/_pagefind"
diff --git a/public/07d125e7231655dd8670b5fb9f6f9aed.txt b/public/07d125e7231655dd8670b5fb9f6f9aed.txt
new file mode 100644
index 0000000..1270078
--- /dev/null
+++ b/public/07d125e7231655dd8670b5fb9f6f9aed.txt
@@ -0,0 +1 @@
+07d125e7231655dd8670b5fb9f6f9aed
diff --git a/public/llms.txt b/public/llms.txt
new file mode 100644
index 0000000..1bdc3ef
--- /dev/null
+++ b/public/llms.txt
@@ -0,0 +1,31 @@
+# zig-nostr
+
+> A native Nostr ecosystem in Zig: `nostr`, a protocol library for Zig; Plaza, a local-first client for macOS and Linux; Notary, a NIP-46 remote signer; and deed, a command line for nostr. MIT licensed, all on one core.
+
+Every page on this site is also served as Markdown at its path with `.md` added (the home page is /index.md), and /llms-full.txt has them all in one file.
+
+## Projects
+
+- [nostr, the Zig library](/getting-started.md): keys and BIP-340 signatures via libsecp256k1, NIP-01 events, NIP-19, NIP-44, NIP-46, NIP-42, NIP-49, NIP-06, relay transport with deadlines, the NIP-65 outbox model, and a memory-mapped LMDB event store. Source: https://github.com/zig-nostr/nostr
+- [deed, the nostr command line](/deed.md): keys, signing, verifying, NIP-19, NIP-44, relay queries, publishing, and a local store it can query offline. One binary for macOS and Linux. Source: https://github.com/zig-nostr/deed
+- [Plaza](/plaza.md): a fast, local-first nostr client for macOS and Linux, rendered from its own store. Source: https://github.com/zig-nostr/plaza
+- [Notary](/notary.md): a native NIP-46 remote signer for macOS and Linux; the key stays in a local daemon on a machine you control. Source: https://github.com/zig-nostr/notary
+
+## For agents
+
+- [Skill: using deed](https://github.com/zig-nostr/deed/blob/main/skills/deed/SKILL.md): install with `npx skills add zig-nostr/deed`. Covers every command, recipes, exit codes, and two rules: publishing is public and permanent, so ask the user first, and keep secret keys in `NOSTR_SECRET_KEY`.
+- [Skill: building with the Zig library](https://github.com/zig-nostr/nostr/blob/main/skills/zig-nostr/SKILL.md): install with `npx skills add zig-nostr/nostr`. Adding the dependency, keys, events, NIP-19, NIP-44, relays read with a deadline, the store, and the traps.
+
+## Docs
+
+- [Getting started](/getting-started.md): add the library and sign a first event
+- [Ecosystem](/ecosystem.md): what exists and how the pieces fit
+- [Architecture](/architecture.md): how the library is put together
+- [NIP support](/nips.md): which NIPs are implemented, and where
+- [Performance](/performance.md): store benchmarks for the library, and deed's numbers
+- [Roadmap](/roadmap.md): what comes next
+
+## Optional
+
+- [Everything in one file](/llms-full.txt)
+- [deed benchmarks](https://github.com/zig-nostr/deed/blob/main/BENCHMARKS.md)
diff --git a/scripts/agent-files.mjs b/scripts/agent-files.mjs
new file mode 100644
index 0000000..4403307
--- /dev/null
+++ b/scripts/agent-files.mjs
@@ -0,0 +1,61 @@
+// Writes the files agents and answer engines read, from the same MDX the site
+// renders, so they can never drift from the pages:
+//
+//   public/<page>.md     a plain Markdown copy of every page (index.md for home)
+//   public/llms-full.txt every page in one file, in navigation order
+//
+// Runs before `next build`. The output is generated, so it is ignored by git.
+import { readFileSync, writeFileSync, readdirSync } from 'node:fs'
+import { join } from 'node:path'
+
+const root = new URL('..', import.meta.url).pathname
+const content = join(root, 'content')
+const out = join(root, 'public')
+
+// Navigation order comes from _meta.js, so llms-full.txt reads like the site.
+const meta = readFileSync(join(content, '_meta.js'), 'utf8')
+const order = [...meta.matchAll(/^\s{2}(?:'([^']+)'|(\w+)):/gm)].map((m) => m[1] || m[2])
+const pages = readdirSync(content).filter((f) => f.endsWith('.mdx')).map((f) => f.slice(0, -4))
+const ordered = [...order.filter((p) => pages.includes(p)), ...pages.filter((p) => !order.includes(p))]
+
+const ELEMENT = '(?:div|span|p|img|h[1-6]|a|br|section|ul|ol|li|strong|em|b|i|code|pre|small)'
+
+function toMarkdown(src) {
+  let title = null
+  src = src.replace(/^---\n([\s\S]*?)\n---\n/, (_, fm) => {
+    const t = fm.match(/^title:\s*(.+)$/m)
+    if (t) title = t[1].trim()
+    return ''
+  })
+  // Protect code, fenced and inline, from the tag stripping below.
+  const kept = []
+  const keep = (s) => `\u0000${kept.push(s) - 1}\u0000`
+  src = src.replace(/```[\s\S]*?```/g, keep).replace(/`[^`\n]+`/g, keep)
+
+  src = src
+    .replace(/^(?:import|export) .*$/gm, '')
+    .replace(/<h([1-6])[^>]*>([\s\S]*?)<\/h\1>/g, (_, n, t) => `\n\n${'#'.repeat(+n)} ${t.trim()}\n\n`)
+    .replace(/<a\b[^>]*?href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/g, (_, href, t) => `[${t.trim()}](${href})`)
+    .replace(/<p\b[^>]*>([\s\S]*?)<\/p>/g, (_, t) => `\n\n${t.trim().replace(/\s*\n\s*/g, ' ')}\n\n`)
+    .replace(new RegExp(`</?${ELEMENT}\\b[^>]*?/?>`, 'g'), '\n')
+    .replace(/&nbsp;/g, ' ')
+    .replace(/&amp;/g, '&')
+    .replace(/[ \t]+$/gm, '')
+    // Indentation survives only on nested list items; anywhere else four
+    // spaces would turn a line into a code block.
+    .replace(/^[ \t]+(?=\S)/gm, (ws, off, all) => (/^[ \t]+(?:[-*+]|\d+\.)\s/.test(all.slice(off, off + ws.length + 4)) ? ws : ''))
+    .replace(/\n{3,}/g, '\n\n')
+    .trim()
+  src = src.replace(/\u0000(\d+)\u0000/g, (_, i) => kept[+i])
+  if (title && !/^# /.test(src)) src = `# ${title}\n\n${src}`
+  return src + '\n'
+}
+
+const full = ['# zig-nostr, every page\n', 'The whole site as Markdown, in navigation order. Each page is also served on its own at its path with `.md` added; the home page is `/index.md`.\n']
+for (const page of ordered) {
+  const md = toMarkdown(readFileSync(join(content, `${page}.mdx`), 'utf8'))
+  writeFileSync(join(out, `${page}.md`), md)
+  full.push(`\n---\n\nSource: /${page === 'index' ? '' : page}\n\n${md}`)
+}
+writeFileSync(join(out, 'llms-full.txt'), full.join(''))
+console.log(`agent files: ${ordered.length} pages, llms-full.txt`)