Add agent-readable docs discovery

apps/docs/astro.config.mjs

@@ -2,6 +2,7 @@
 import { defineConfig } from "astro/config";
 import { unified } from "@astrojs/markdown-remark";
 import starlight from "@astrojs/starlight";
+import starlightLlmsTxt from "starlight-llms-txt";
 import remarkDocLinks from "./remark-doc-links.mjs";
 import remarkMermaid from "./remark-mermaid.mjs";
 
@@ -72,6 +73,24 @@ export default defineConfig({
       },
       // Collapse the now-empty Starlight title panel left by the PageTitle override.
       customCss: ["./src/styles/docs.css"],
+      plugins: [
+        starlightLlmsTxt({
+          projectName: "Rakkr",
+          description:
+            "Documentation for Rakkr, a local-only, centrally managed Linux audio recording platform for reliable room recording.",
+          details:
+            "Use the public documentation sets below for installation, operation, architecture, and development guidance. Internal status ledgers and verification baselines are intentionally excluded.",
+          promote: ["index*", "getting-started/**", "how-to/**"],
+          demote: ["contributing/**"],
+          optionalLinks: [
+            {
+              label: "Rakkr source repository",
+              url: "https://github.com/yashau/Rakkr",
+              description: "source code, issue tracking, and release history",
+            },
+          ],
+        }),
+      ],
       social: [
         {
           icon: "github",

apps/docs/eslint.config.mjs

@@ -4,7 +4,7 @@ import eslintPluginAstro from "eslint-plugin-astro";
 export default [
   // The Cloudflare Worker is TypeScript with Workers globals; it is typechecked
   // via worker/tsconfig.json and bundled by wrangler, not by this Astro lint.
-  { ignores: ["dist/**", ".astro/**", "worker/**"] },
+  { ignores: ["dist/**", ".astro/**", ".wrangler/**", "worker/**"] },
   js.configs.recommended,
   ...eslintPluginAstro.configs["flat/recommended"],
 ];

apps/docs/package.json

@@ -5,7 +5,7 @@
   "type": "module",
   "scripts": {
     "dev": "astro dev",
-    "build": "astro build",
+    "build": "astro build && node scripts/generate-agent-markdown.mjs && node scripts/verify-discovery.mjs",
     "preview": "astro preview",
     "check": "astro check",
     "lint": "eslint .",
@@ -16,15 +16,20 @@
     "@astrojs/markdown-remark": "^7.2.2",
     "@astrojs/starlight": "^0.41.6",
     "astro": "^7.1.6",
-    "mermaid": "^11.16.0"
+    "mermaid": "^11.16.0",
+    "starlight-llms-txt": "^0.11.0"
   },
   "devDependencies": {
     "@astrojs/check": "^0.9.10",
     "@cloudflare/workers-types": "^5.20260801.1",
     "@eslint/js": "^10.0.1",
     "eslint": "^10.8.0",
     "eslint-plugin-astro": "^3.1.0",
+    "remark-frontmatter": "^5.0.0",
+    "remark-parse": "^11.0.0",
+    "remark-stringify": "^11.0.0",
     "typescript": "^6.0.3",
+    "unified": "^11.0.5",
     "wrangler": "^4.118.0"
   }
 }

apps/docs/public/robots.txt

@@ -0,0 +1,6 @@
+User-agent: *
+Allow: /
+
+Sitemap: https://docs.rakkr.org/sitemap-index.xml
+
+# Machine-readable documentation: https://docs.rakkr.org/llms.txt

apps/docs/remark-doc-links.mjs

@@ -12,11 +12,11 @@ const DOCS_ROOT = path.resolve(import.meta.dirname, "../../docs");
 const REPO_ROOT = path.resolve(import.meta.dirname, "../..");
 const GITHUB_BASE = "https://github.com/yashau/Rakkr";
 
-function isExcluded(relFromDocs) {
+export function isExcluded(relFromDocs) {
   return relFromDocs === "RAKKR_SOURCE_OF_TRUTH.md" || relFromDocs.startsWith("internal/");
 }
 
-function toSlug(relFromDocs) {
+export function toSlug(relFromDocs) {
   const noExt = relFromDocs.replace(/\.(md|mdx)$/i, "");
   const normalized = noExt.replace(/(^|\/)readme$/i, "$1index");
   const trimmed = normalized.replace(/\/index$/i, "");

apps/docs/scripts/generate-agent-markdown.mjs

@@ -0,0 +1,47 @@
+import { mkdir, readFile, readdir, writeFile } from "node:fs/promises";
+import path from "node:path";
+import remarkFrontmatter from "remark-frontmatter";
+import remarkParse from "remark-parse";
+import remarkStringify from "remark-stringify";
+import { unified } from "unified";
+import remarkDocLinks, { isExcluded, toSlug } from "../remark-doc-links.mjs";
+
+const docsRoot = path.resolve(import.meta.dirname, "../../../docs");
+const distRoot = path.resolve(import.meta.dirname, "../dist");
+
+const processor = unified()
+  .use(remarkParse)
+  .use(remarkFrontmatter, ["yaml"])
+  .use(remarkDocLinks)
+  .use(remarkStringify, { bullet: "-", fences: true });
+
+for (const sourcePath of await markdownFiles(docsRoot)) {
+  const relativePath = toPosix(path.relative(docsRoot, sourcePath));
+  if (isExcluded(relativePath)) continue;
+
+  const source = await readFile(sourcePath, "utf8");
+  const markdown = String(await processor.process({ path: sourcePath, value: source }));
+  const outputPath = path.join(distRoot, toSlug(relativePath), "index.md");
+
+  await mkdir(path.dirname(outputPath), { recursive: true });
+  await writeFile(outputPath, markdown, "utf8");
+}
+
+async function markdownFiles(directory) {
+  const paths = [];
+
+  for (const entry of await readdir(directory, { withFileTypes: true })) {
+    const entryPath = path.join(directory, entry.name);
+    if (entry.isDirectory()) {
+      paths.push(...(await markdownFiles(entryPath)));
+    } else if (/\.(md|mdx)$/iu.test(entry.name)) {
+      paths.push(entryPath);
+    }
+  }
+
+  return paths;
+}
+
+function toPosix(value) {
+  return value.split(path.sep).join("/");
+}

apps/docs/scripts/verify-discovery.mjs

@@ -0,0 +1,61 @@
+import assert from "node:assert/strict";
+import { readFile, readdir } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath, URL } from "node:url";
+
+const dist = new URL("../dist/", import.meta.url);
+const readDistFile = (path) => readFile(new URL(path, dist), "utf8");
+
+const [robots, llms, llmsFull, llmsSmall, sitemapIndex, sitemap, pageMarkdown] = await Promise.all([
+  readDistFile("robots.txt"),
+  readDistFile("llms.txt"),
+  readDistFile("llms-full.txt"),
+  readDistFile("llms-small.txt"),
+  readDistFile("sitemap-index.xml"),
+  readDistFile("sitemap-0.xml"),
+  readDistFile("architecture/overview/index.md"),
+]);
+
+assert.match(robots, /^User-agent: \*$/mu);
+assert.match(robots, /^Allow: \/$/mu);
+assert.match(robots, /^Sitemap: https:\/\/docs\.rakkr\.org\/sitemap-index\.xml$/mu);
+assert.match(robots, /https:\/\/docs\.rakkr\.org\/llms\.txt/u);
+
+assert.match(llms, /^# Rakkr$/mu);
+assert.match(llms, /https:\/\/docs\.rakkr\.org\/llms-small\.txt/u);
+assert.match(llms, /https:\/\/docs\.rakkr\.org\/llms-full\.txt/u);
+assert.match(llmsFull, /^# Rakkr Documentation$/mu);
+assert.match(llmsSmall, /^# Rakkr Documentation$/mu);
+
+assert.match(sitemapIndex, /https:\/\/docs\.rakkr\.org\/sitemap-0\.xml/u);
+assert.match(sitemap, /<loc>https:\/\/docs\.rakkr\.org\/<\/loc>/u);
+assert.match(sitemap, /<loc>https:\/\/docs\.rakkr\.org\/getting-started\/introduction\/<\/loc>/u);
+assert.doesNotMatch(sitemap, /\/internal\//u);
+assert.doesNotMatch(sitemap, /RAKKR_SOURCE_OF_TRUTH/u);
+
+assert.match(pageMarkdown, /^title: Architecture overview$/mu);
+assert.match(pageMarkdown, /\[Controller API\]\(\/architecture\/controller-api\/\)/u);
+assert.doesNotMatch(pageMarkdown, /\]\(controller-api\.md\)/u);
+
+const sitemapPageCount = [...sitemap.matchAll(/<loc>/gu)].length;
+const markdownPaths = await findNamedFiles(fileURLToPath(dist), "index.md");
+assert.equal(markdownPaths.length, sitemapPageCount);
+assert.equal(
+  markdownPaths.some((file) => file.includes(`${path.sep}internal${path.sep}`)),
+  false,
+);
+
+async function findNamedFiles(directory, fileName) {
+  const paths = [];
+
+  for (const entry of await readdir(directory, { withFileTypes: true })) {
+    const entryPath = path.join(directory, entry.name);
+    if (entry.isDirectory()) {
+      paths.push(...(await findNamedFiles(entryPath, fileName)));
+    } else if (entry.name === fileName) {
+      paths.push(entryPath);
+    }
+  }
+
+  return paths;
+}

apps/docs/worker/index.ts

@@ -4,9 +4,10 @@
 //
 // The Starlight build is uploaded as static assets (see `wrangler.jsonc`).
 // Cloudflare serves a matching asset before invoking this Worker, so the handler
-// only runs for non-asset routes. It answers `/version.json` (so the deployed
-// release is verifiable from the edge) and delegates everything else back to the
-// static assets, including Starlight's generated `404.html`.
+// runs first for canonical page paths so agents can negotiate Markdown, while
+// CSS, JavaScript, images, and other static files still use asset-first routing.
+// It also answers `/version.json` so the deployed release is verifiable from the
+// edge and delegates everything else back to the static assets.
 
 interface Env {
   ASSETS: Fetcher;
@@ -28,6 +29,56 @@ export default {
       );
     }
 
+    if (acceptsMarkdown(request) && (request.method === "GET" || request.method === "HEAD")) {
+      const markdownResponse = await getPageMarkdown(request, env, url);
+      if (markdownResponse) return markdownResponse;
+    }
+
     return env.ASSETS.fetch(request);
   },
 } satisfies ExportedHandler<Env>;
+
+function acceptsMarkdown(request: Request): boolean {
+  const accept = request.headers.get("accept");
+  if (!accept) return false;
+
+  return accept.split(",").some((range) => {
+    const [mediaType, ...parameters] = range.split(";").map((part) => part.trim().toLowerCase());
+    if (mediaType !== "text/markdown") return false;
+
+    const quality = parameters.find((parameter) => parameter.startsWith("q="));
+    return quality ? Number(quality.slice(2)) > 0 : true;
+  });
+}
+
+async function getPageMarkdown(
+  request: Request,
+  env: Env,
+  url: URL,
+): Promise<Response | undefined> {
+  if (!url.pathname.endsWith("/")) return;
+
+  const markdownUrl = new URL(`${url.pathname}index.md`, url);
+  const assetResponse = await env.ASSETS.fetch(
+    new Request(markdownUrl, { method: request.method, headers: request.headers }),
+  );
+  if (!assetResponse.ok) return;
+
+  const headers = new Headers(assetResponse.headers);
+  headers.set("content-type", "text/markdown; charset=utf-8");
+  headers.set("content-signal", "ai-train=yes, search=yes, ai-input=yes");
+  headers.set("vary", appendVary(headers.get("vary"), "Accept"));
+
+  return new Response(request.method === "HEAD" ? null : assetResponse.body, {
+    status: assetResponse.status,
+    statusText: assetResponse.statusText,
+    headers,
+  });
+}
+
+function appendVary(current: string | null, value: string): string {
+  const values = current?.split(",").map((entry) => entry.trim()) ?? [];
+  if (values.includes("*")) return "*";
+  if (!values.some((entry) => entry.toLowerCase() === value.toLowerCase())) values.push(value);
+  return values.join(", ");
+}

apps/docs/wrangler.jsonc

@@ -13,6 +13,10 @@
   "assets": {
     "directory": "./dist",
     "binding": "ASSETS",
+    // Invoke the Worker only for canonical page URLs. This enables
+    // `Accept: text/markdown` negotiation without putting hashed CSS, JS, image,
+    // sitemap, robots, or llms.txt asset requests on the Worker invocation path.
+    "run_worker_first": ["/", "/*/"],
   },
   // Overridden at deploy time with the release version: `wrangler deploy --var
   // RAKKR_DOCS_VERSION:<YYYY.MM.DD-N> --var RAKKR_DOCS_COMMIT:<sha>`.

docs/operations/releases.md

@@ -75,6 +75,30 @@ Worker as a variable and is verifiable at `https://docs.rakkr.org/version.json`.
 There is no GitHub release — Cloudflare stores the deployment. Requires the
 `CLOUDFLARE_API_TOKEN` and `CLOUDFLARE_ACCOUNT_ID` repository secrets.
 
+The docs build also publishes machine-readable discovery artifacts:
+
+- `/llms.txt` points agents to complete and abridged Markdown documentation sets;
+- `/llms-full.txt` and `/llms-small.txt` are generated from the same public
+  Starlight content collection as the HTML site;
+- each public page has a source-aligned Markdown asset at its trailing
+  `/index.md` path;
+- `/robots.txt` allows crawling and advertises `/sitemap-index.xml`;
+- Starlight generates `/sitemap-index.xml` and its sitemap shard from the public
+  routes.
+
+The docs Worker implements the **Markdown for Agents** content-negotiation
+contract directly, so it also works on Cloudflare's Free plan: a request with
+`Accept: text/markdown` returns the matching page's generated Markdown with
+`Content-Type: text/markdown`, `Vary: Accept`, and the site's content-use signal.
+Static scripts, styles, images, sitemaps, and discovery files remain on
+Cloudflare's asset-first path. Verify both discovery paths after a docs release:
+
+```powershell
+Invoke-WebRequest https://docs.rakkr.org/llms.txt
+Invoke-WebRequest https://docs.rakkr.org/getting-started/introduction/ `
+  -Headers @{ Accept = "text/markdown" }
+```
+
 ### Controller (`controller-v*`)
 
 `release-controller.yml` builds and pushes versioned images to GHCR: