diff --git a/packages/astro/src/server.ts b/packages/astro/src/server.ts index 49441878..1ed681e6 100644 --- a/packages/astro/src/server.ts +++ b/packages/astro/src/server.ts @@ -42,6 +42,7 @@ import { emitDocsAnalyticsEvent, formatDocsAskAIPackageHints, findDocsMarkdownPage, + getDocsMarkdownVaryHeader, isDocsAgentDiscoveryRequest, isDocsSkillRequest, normalizeDocsRelated, @@ -896,12 +897,14 @@ export function createDocsServer(config: Record = {}): DocsServer { const markdownRequest = resolveDocsMarkdownRequest(entry, url, context.request); if (markdownRequest) { const document = getMarkdownDocument(ctx, markdownRequest.requestedPath); + const varyHeader = getDocsMarkdownVaryHeader(context.request); if (!document) { return new Response("Not Found", { status: 404, headers: { "Content-Type": "text/plain; charset=utf-8", + ...(varyHeader ? { Vary: varyHeader } : {}), "X-Robots-Tag": "noindex", }, }); @@ -911,6 +914,7 @@ export function createDocsServer(config: Record = {}): DocsServer { headers: { "Content-Type": "text/markdown; charset=utf-8", "Cache-Control": "public, max-age=0, s-maxage=3600", + ...(varyHeader ? { Vary: varyHeader } : {}), "X-Robots-Tag": "noindex", }, }); diff --git a/packages/docs/src/agent.test.ts b/packages/docs/src/agent.test.ts index 92b7532f..0b2d785d 100644 --- a/packages/docs/src/agent.test.ts +++ b/packages/docs/src/agent.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { buildDocsAgentDiscoverySpec, findDocsMarkdownPage, + getDocsMarkdownVaryHeader, hasDocsMarkdownSignatureAgent, isDocsAgentDiscoveryRequest, isDocsMcpRequest, @@ -135,6 +136,21 @@ describe("agent route helpers", () => { }), ), ).toBe(true); + expect( + getDocsMarkdownVaryHeader( + new Request("https://example.com/docs/install", { + headers: { accept: "text/markdown" }, + }), + ), + ).toBe("Accept"); + expect( + getDocsMarkdownVaryHeader( + new Request("https://example.com/docs/install", { + headers: { "Signature-Agent": "https://chatgpt.com" }, + }), + ), + ).toBe("Accept, Signature-Agent"); + expect(getDocsMarkdownVaryHeader(new Request("https://example.com/docs/install"))).toBeNull(); const apiFormatRoute = resolveDocsMarkdownRequest( "docs", diff --git a/packages/docs/src/agent.ts b/packages/docs/src/agent.ts index 73bc2631..4817293c 100644 --- a/packages/docs/src/agent.ts +++ b/packages/docs/src/agent.ts @@ -222,6 +222,14 @@ export function hasDocsMarkdownSignatureAgent(request: Request): boolean { return Boolean(request.headers.get(DOCS_MARKDOWN_SIGNATURE_AGENT_HEADER)?.trim()); } +export function getDocsMarkdownVaryHeader(request: Request): string | null { + if (hasDocsMarkdownSignatureAgent(request)) { + return `Accept, ${DOCS_MARKDOWN_SIGNATURE_AGENT_HEADER}`; + } + + return acceptsMarkdown(request) ? "Accept" : null; +} + export function findDocsMarkdownPage( entry: string, pages: T[], diff --git a/packages/docs/src/index.ts b/packages/docs/src/index.ts index 0ecbbb71..07e68094 100644 --- a/packages/docs/src/index.ts +++ b/packages/docs/src/index.ts @@ -67,6 +67,7 @@ export { DOCS_MARKDOWN_SIGNATURE_AGENT_HEADER, buildDocsAgentDiscoverySpec, findDocsMarkdownPage, + getDocsMarkdownVaryHeader, hasDocsMarkdownSignatureAgent, isDocsAgentDiscoveryRequest, isDocsMcpRequest, diff --git a/packages/fumadocs/src/docs-api.test.ts b/packages/fumadocs/src/docs-api.test.ts index 90b4dd61..7512fa53 100644 --- a/packages/fumadocs/src/docs-api.test.ts +++ b/packages/fumadocs/src/docs-api.test.ts @@ -641,6 +641,7 @@ Config content. ); expect(fallbackResponse.status).toBe(200); expect(fallbackResponse.headers.get("content-type")).toContain("text/markdown"); + expect(fallbackResponse.headers.get("vary")).toBeNull(); const fallbackDocument = await fallbackResponse.text(); expect(fallbackDocument).toContain("# Quickstart\nURL: /docs/getting-started/quickstart"); expect(fallbackDocument).toContain( @@ -662,6 +663,7 @@ Config content. ); expect(rewrittenFallbackResponse.status).toBe(200); expect(rewrittenFallbackResponse.headers.get("content-type")).toContain("text/markdown"); + expect(rewrittenFallbackResponse.headers.get("vary")).toBeNull(); expect(await rewrittenFallbackResponse.text()).toContain( "Verify the onboarding command examples before changing this page.", ); @@ -677,6 +679,7 @@ Config content. ); expect(acceptFallbackResponse.status).toBe(200); expect(acceptFallbackResponse.headers.get("content-type")).toContain("text/markdown"); + expect(acceptFallbackResponse.headers.get("vary")).toBe("Accept"); expect(await acceptFallbackResponse.text()).toContain( "Verify the onboarding command examples before changing this page.", ); @@ -687,6 +690,7 @@ Config content. }), ); expect(acceptAgentResponse.status).toBe(200); + expect(acceptAgentResponse.headers.get("vary")).toBe("Accept"); expect(await acceptAgentResponse.text()).toBe("Use this page as the implementation map.\n"); const weightedAcceptAgentResponse = await GET( @@ -696,10 +700,20 @@ Config content. ); expect(weightedAcceptAgentResponse.status).toBe(200); expect(weightedAcceptAgentResponse.headers.get("content-type")).toContain("text/markdown"); + expect(weightedAcceptAgentResponse.headers.get("vary")).toBe("Accept"); expect(await weightedAcceptAgentResponse.text()).toBe( "Use this page as the implementation map.\n", ); + const signatureAgentResponse = await GET( + new Request("http://localhost/api/docs?format=markdown&path=overview", { + headers: { "Signature-Agent": "https://chatgpt.com" }, + }), + ); + expect(signatureAgentResponse.status).toBe(200); + expect(signatureAgentResponse.headers.get("vary")).toBe("Accept, Signature-Agent"); + expect(await signatureAgentResponse.text()).toBe("Use this page as the implementation map.\n"); + const zeroQualityAcceptResponse = await GET( new Request("http://localhost/docs/overview", { headers: { accept: "application/json, text/markdown;profile=agent;q=0" }, diff --git a/packages/fumadocs/src/docs-api.ts b/packages/fumadocs/src/docs-api.ts index 36f38b76..04f291fc 100644 --- a/packages/fumadocs/src/docs-api.ts +++ b/packages/fumadocs/src/docs-api.ts @@ -31,6 +31,7 @@ import { createDocsAgentTraceId, emitDocsAgentTraceEvent, emitDocsAnalyticsEvent, + getDocsMarkdownVaryHeader, resolveDocsI18n, resolveDocsLocale, resolvePageSidebarFolderIndexBehavior, @@ -2690,6 +2691,7 @@ export function createDocsAPI(options?: DocsAPIOptions) { if (markdownRequest) { const document = await getMarkdownDocument(ctx, markdownRequest.requestedPath); + const varyHeader = getDocsMarkdownVaryHeader(request); if (!document) { await emitDocsAnalyticsEvent(analytics, { @@ -2720,6 +2722,7 @@ export function createDocsAPI(options?: DocsAPIOptions) { status: 404, headers: { "Content-Type": "text/plain; charset=utf-8", + ...(varyHeader ? { Vary: varyHeader } : {}), "X-Robots-Tag": "noindex", }, }); @@ -2755,6 +2758,7 @@ export function createDocsAPI(options?: DocsAPIOptions) { headers: { "Content-Type": "text/markdown; charset=utf-8", "Cache-Control": "public, max-age=0, s-maxage=3600", + ...(varyHeader ? { Vary: varyHeader } : {}), "X-Robots-Tag": "noindex", }, }); diff --git a/packages/nuxt/src/server.ts b/packages/nuxt/src/server.ts index 474057cf..ebbe7745 100644 --- a/packages/nuxt/src/server.ts +++ b/packages/nuxt/src/server.ts @@ -29,6 +29,7 @@ import { emitDocsAnalyticsEvent, formatDocsAskAIPackageHints, findDocsMarkdownPage, + getDocsMarkdownVaryHeader, isDocsAgentDiscoveryRequest, isDocsMcpRequest, isDocsPublicGetRequest, @@ -885,12 +886,14 @@ export function createDocsServer(config: Record = {}): DocsServer { const markdownRequest = resolveDocsMarkdownRequest(entry, url, context.request); if (markdownRequest) { const document = getMarkdownDocument(ctx, markdownRequest.requestedPath); + const varyHeader = getDocsMarkdownVaryHeader(context.request); if (!document) { return new Response("Not Found", { status: 404, headers: { "Content-Type": "text/plain; charset=utf-8", + ...(varyHeader ? { Vary: varyHeader } : {}), "X-Robots-Tag": "noindex", }, }); @@ -900,6 +903,7 @@ export function createDocsServer(config: Record = {}): DocsServer { headers: { "Content-Type": "text/markdown; charset=utf-8", "Cache-Control": "public, max-age=0, s-maxage=3600", + ...(varyHeader ? { Vary: varyHeader } : {}), "X-Robots-Tag": "noindex", }, }); diff --git a/packages/svelte/src/server.ts b/packages/svelte/src/server.ts index 2025b688..19fe96c5 100644 --- a/packages/svelte/src/server.ts +++ b/packages/svelte/src/server.ts @@ -42,6 +42,7 @@ import { emitDocsAnalyticsEvent, formatDocsAskAIPackageHints, findDocsMarkdownPage, + getDocsMarkdownVaryHeader, isDocsAgentDiscoveryRequest, isDocsSkillRequest, normalizeDocsRelated, @@ -904,12 +905,14 @@ export function createDocsServer(config: Record = {}): DocsServer { const markdownRequest = resolveDocsMarkdownRequest(entry, event.url, event.request); if (markdownRequest) { const document = getMarkdownDocument(ctx, markdownRequest.requestedPath); + const varyHeader = getDocsMarkdownVaryHeader(event.request); if (!document) { return new Response("Not Found", { status: 404, headers: { "Content-Type": "text/plain; charset=utf-8", + ...(varyHeader ? { Vary: varyHeader } : {}), "X-Robots-Tag": "noindex", }, }); @@ -919,6 +922,7 @@ export function createDocsServer(config: Record = {}): DocsServer { headers: { "Content-Type": "text/markdown; charset=utf-8", "Cache-Control": "public, max-age=0, s-maxage=3600", + ...(varyHeader ? { Vary: varyHeader } : {}), "X-Robots-Tag": "noindex", }, }); diff --git a/packages/tanstack-start/src/server.ts b/packages/tanstack-start/src/server.ts index 25ec75ff..b0617d6d 100644 --- a/packages/tanstack-start/src/server.ts +++ b/packages/tanstack-start/src/server.ts @@ -12,6 +12,7 @@ import { emitDocsAnalyticsEvent, formatDocsAskAIPackageHints, findDocsMarkdownPage, + getDocsMarkdownVaryHeader, isDocsAgentDiscoveryRequest, isDocsSkillRequest, normalizeDocsRelated, @@ -869,12 +870,14 @@ export function createDocsServer(config: Record): DocsServer { const markdownRequest = resolveDocsMarkdownRequest(entry, url, event.request); if (markdownRequest) { const document = getMarkdownDocument(ctx, markdownRequest.requestedPath); + const varyHeader = getDocsMarkdownVaryHeader(event.request); if (!document) { return new Response("Not Found", { status: 404, headers: { "Content-Type": "text/plain; charset=utf-8", + ...(varyHeader ? { Vary: varyHeader } : {}), "X-Robots-Tag": "noindex", }, }); @@ -884,6 +887,7 @@ export function createDocsServer(config: Record): DocsServer { headers: { "Content-Type": "text/markdown; charset=utf-8", "Cache-Control": "public, max-age=0, s-maxage=3600", + ...(varyHeader ? { Vary: varyHeader } : {}), "X-Robots-Tag": "noindex", }, }); diff --git a/website/app/docs/configuration/page.mdx b/website/app/docs/configuration/page.mdx index b04a8193..ee1e079d 100644 --- a/website/app/docs/configuration/page.mdx +++ b/website/app/docs/configuration/page.mdx @@ -181,6 +181,18 @@ No separate config flag is required for machine-readable page markdown. In Next.js, browsers still receive HTML from `/docs/installation`; agents, scripts, and crawlers that send `Accept: text/markdown` or `Signature-Agent` receive the same machine-readable output as `/docs/installation.md`. Other adapters should use the `.md` URL or the API format route. + + Negotiated markdown responses include cache-safe `Vary` headers: `Accept` for + `Accept: text/markdown`, and `Accept, Signature-Agent` for `Signature-Agent`. + + That means a CDN can cache the HTML and markdown versions separately even though the URL is the + same: + + ```txt + /docs/installation + normal browser headers -> cached HTML + /docs/installation + Accept: text/markdown -> cached markdown + /docs/installation + Signature-Agent: -> cached markdown + ``` ```mdx title="page.mdx" diff --git a/website/app/docs/customization/agent-primitive/page.mdx b/website/app/docs/customization/agent-primitive/page.mdx index 44f70cc5..337b136e 100644 --- a/website/app/docs/customization/agent-primitive/page.mdx +++ b/website/app/docs/customization/agent-primitive/page.mdx @@ -28,6 +28,10 @@ The same page model also powers the built-in machine-readable routes: `Accept: text/markdown` or `Signature-Agent` to that same URL when an agent or script wants the machine-readable version without appending `.md`. In the other adapters, use `/docs/installation.md` or the `format=markdown` API route. + + The markdown response varies by the relevant request headers, so CDNs do not mix the HTML and + markdown representations for the same URL. A browser can keep receiving cached HTML while an + agent receives cached markdown for `/docs/installation`. Use `Agent` when the human page is already mostly correct and just needs a little extra implementation