diff --git a/apps/e2e-test/landing/docs.spec.ts b/apps/e2e-test/landing/docs.spec.ts index e2de929af..70ac3b1aa 100644 --- a/apps/e2e-test/landing/docs.spec.ts +++ b/apps/e2e-test/landing/docs.spec.ts @@ -319,4 +319,99 @@ test.describe('docs', () => { await expect(vueTabAgain).toHaveAttribute('aria-selected', 'true') }).toPass({ timeout: 15_000 }) }) + + // ── SEO surfaces ────────────────────────────────────────────────────── + // The dev server this project boots leaves NEXT_PUBLIC_BASE_URL unset, so + // next.config.mjs computes SITE_BASE = https://useupup.com and takes the + // PRODUCTION branch of headers()/redirects(). That is what lets these + // assertions exercise the real prod rules without a second webServer. + const PRODUCTION_ORIGIN = 'https://useupup.com' + + test('docs page declares its canonical URL and its markdown twin as an alternate', async ({ + page, + request, + }) => { + await page.goto('/docs/getting-started/') + await expect(page.locator('link[rel=canonical]')).toHaveAttribute( + 'href', + `${PRODUCTION_ORIGIN}/docs/getting-started/`, + ) + const alternate = page.locator( + 'link[rel=alternate][type="text/markdown"]', + ) + await expect(alternate).toHaveAttribute( + 'href', + `${PRODUCTION_ORIGIN}/docs-md/getting-started/`, + ) + // Fetch the PATH against this server — following the absolute href + // would test production, not the build under test. + const href = await alternate.getAttribute('href') + const twin = await request.get(new URL(href ?? '').pathname) + expect(twin.status()).toBe(200) + expect(twin.headers()['content-type']).toContain('text/markdown') + // The twin must declare the HTML page as its original, or it is a + // duplicate of every docs page under a second URL. + expect(twin.headers()['link']).toContain('rel="canonical"') + expect(twin.headers()['link']).toContain( + `${PRODUCTION_ORIGIN}/docs/getting-started/`, + ) + }) + + test('plaintext request identified by the Cloudflare visitor header is redirected to https', async ({ + request, + }) => { + const redirected = await request.get('/react/', { + headers: { 'cf-visitor': '{"scheme":"http"}' }, + maxRedirects: 0, + }) + expect(redirected.status()).toBe(308) + expect(redirected.headers()['location']).toBe( + `${PRODUCTION_ORIGIN}/react/`, + ) + // The header is the ONLY trigger: an ordinary request must still be + // served, or the rule would loop every visitor behind the proxy. + const plain = await request.get('/react/') + expect(plain.status()).toBe(200) + }) + + test('stale search-console sitemap URL permanently redirects to the live sitemap', async ({ + request, + }) => { + // /sitemap-landing.xml is a Docusaurus-era submission that nothing has + // ever served; it 404'd until this rule landed. + const res = await request.get('/sitemap-landing.xml', { + maxRedirects: 0, + }) + expect(res.status()).toBe(308) + expect(res.headers()['location']).toContain('/sitemap.xml') + }) + + test('production responses carry a preload-eligible HSTS header', async ({ + request, + }) => { + const res = await request.get('/') + expect(res.headers()['strict-transport-security']).toBe( + 'max-age=63072000; includeSubDomains; preload', + ) + }) + + test('robots.txt names the AI crawler allow-list explicitly', async ({ + request, + }) => { + const res = await request.get('/robots.txt') + expect(res.status()).toBe(200) + const body = await res.text() + expect(body).toContain('User-Agent: GPTBot') + expect(body).toContain('User-Agent: ClaudeBot') + expect(body).toContain('User-Agent: PerplexityBot') + }) + + test('footer links the llms.txt corpus from every page', async ({ + page, + }) => { + await page.goto('/docs/getting-started/') + await expect( + page.locator('footer a[href="/llms.txt"]'), + ).toHaveAttribute('href', '/llms.txt') + }) }) diff --git a/apps/landing/next.config.mjs b/apps/landing/next.config.mjs index 4d3c1b7ca..9614d9686 100644 --- a/apps/landing/next.config.mjs +++ b/apps/landing/next.config.mjs @@ -72,6 +72,46 @@ const nextConfig = { // then one trailingSlash hop appends the slash). async redirects() { return [ + // http -> https, FIRST so no other rule can answer a plaintext + // request with a 200. Keyed on Cloudflare's `cf-visitor` header + // (`{"scheme":"http"}`) and NOTHING else: `x-forwarded-proto` is + // rewritten by Traefik on the way to this container, so a rule + // reading it sees "http" on every request and redirects forever. + // Cloudflare's own "Always Use HTTPS" toggle is the belt (an owner + // item); this is the braces that lives in the repo and survives a + // zone-settings change. + // + // TWO rules, because Next strips the trailing slash before matching + // a source and does NOT re-append it to an ABSOLUTE destination + // (it does for relative ones) — a single `${SITE_BASE}/:path*` + // sends /react/ to `…/react`, costing a second 308. The first rule + // therefore matches extensionless paths only (`[^/.]+` final + // segment) and restores the slash; file paths and the bare root + // fall through to the second, which must NOT gain one. + { + source: '/:path((?:[^/]+/)*[^/.]+)', + has: [ + { + type: 'header', + key: 'cf-visitor', + value: '.*"scheme":"http".*', + }, + ], + destination: `${SITE_BASE}/:path/`, + permanent: true, + }, + { + source: '/:path*', + has: [ + { + type: 'header', + key: 'cf-visitor', + value: '.*"scheme":"http".*', + }, + ], + destination: `${SITE_BASE}/:path*`, + permanent: true, + }, // The wildcard `/documentation/:path*` rule below also covers the // bare path (`:path*` matches zero segments) — this explicit entry // is kept for clarity, not necessity. @@ -121,11 +161,30 @@ const nextConfig = { destination: '/docs/api-reference/upupuploader/required-props/', permanent: true, }, + // Two sitemap URLs still registered in Search Console from the + // Docusaurus era, both currently dead: `/sitemap-landing.xml` is a + // bare 404 (nothing ever served it), and `/documentation/sitemap.xml` + // fell through to the wildcard below, which appends a slash to a + // FILE path and lands on the docs catch-all's 404. GSC has been + // reporting "couldn't fetch" for both ever since. Extension paths + // get no trailing-slash hop, so each of these is a single 308. + // They must precede the wildcard. (Owner follow-up: delete the two + // stale submissions in Search Console once these are live.) + { + source: '/sitemap-landing.xml', + destination: '/sitemap.xml', + permanent: true, + }, + { + source: '/documentation/sitemap.xml', + destination: '/sitemap.xml', + permanent: true, + }, // Destination carries the trailing slash so trailingSlash:true // does not have to spend a SECOND 308 appending it. Safe here only // because every extensionless legacy path maps to a real page and - // the two file paths under /documentation are handled by the - // explicit llms rules above — a slash appended to a file URL would + // the file paths under /documentation are handled by the explicit + // llms + sitemap rules above — a slash appended to a file URL would // break it (Next never slashes paths with an extension). { source: '/documentation/:path*', @@ -137,12 +196,30 @@ const nextConfig = { // /documentation/* -> /docs/* hop first (staying on the alias // host), then the /docs/* rule below moves it to the main host. // - // The three explicit rules come before the two catch-alls because - // a catch-all destination of `/docs/:path*/` is wrong for exactly + // The explicit rules come before the two catch-alls because a + // catch-all destination of `/docs/:path*/` is wrong for exactly // two shapes: an EMPTY `:path*` (which would render `/docs//`) and // a FILE path (which must not gain a trailing slash). Listing them // explicitly lets the catch-alls stay slashed for the page shapes // that are 99% of alias traffic. + // + // robots.txt and sitemap.xml are the file paths a crawler probes on + // any host it meets. Without these two they took the catch-all to + // `${SITE_BASE}/docs/robots.txt/`, then a trailing-slash hop, then + // the docs catch-all's 404 — a broken robots on a live alias host. + // They belong on the APEX copies, not under /docs. + { + source: '/robots.txt', + has: [{ type: 'host', value: DOCS_ALIAS_HOST }], + destination: `${SITE_BASE}/robots.txt`, + permanent: true, + }, + { + source: '/sitemap.xml', + has: [{ type: 'host', value: DOCS_ALIAS_HOST }], + destination: `${SITE_BASE}/sitemap.xml`, + permanent: true, + }, { source: '/llms.txt', has: [{ type: 'host', value: DOCS_ALIAS_HOST }], @@ -200,11 +277,35 @@ const nextConfig = { }, ] }, - // Non-production hosts (dev, previews) serve a byte-identical copy of the - // whole site. robots.txt disallows crawling there; this header is what - // actually keeps a URL discovered some other way out of the index. + // Two mutually exclusive header sets. + // + // PRODUCTION: HSTS. Two years + includeSubDomains + preload is the + // preload-list-eligible value; the site is https-only in practice and the + // cf-visitor redirect above guarantees a plaintext request never gets a + // 200, so there is no http-only subdomain this can strand. (Submitting to + // hstspreload.org is an owner step, taken only after the header has been + // live for a while — `preload` in the value is a prerequisite, not the + // submission itself.) + // + // NON-PRODUCTION (dev, previews): those hosts serve a byte-identical copy + // of the whole site. robots.txt disallows crawling there; this header is + // what actually keeps a URL discovered some other way out of the index. + // They get no HSTS — a preload directive from a preview host is a + // liability, not a protection. async headers() { - if (IS_PRODUCTION_SITE) return [] + if (IS_PRODUCTION_SITE) { + return [ + { + source: '/:path*', + headers: [ + { + key: 'Strict-Transport-Security', + value: 'max-age=63072000; includeSubDomains; preload', + }, + ], + }, + ] + } return [ { source: '/:path*', diff --git a/apps/landing/src/__tests__/seo-surfaces.test.ts b/apps/landing/src/__tests__/seo-surfaces.test.ts new file mode 100644 index 000000000..00cbfd0c3 --- /dev/null +++ b/apps/landing/src/__tests__/seo-surfaces.test.ts @@ -0,0 +1,222 @@ +import { createElement } from 'react' +import { renderToStaticMarkup } from 'react-dom/server' +import { afterEach, describe, expect, it, vi } from 'vitest' +import robots from '@/app/robots' +import sitemap from '@/app/sitemap' +import EntityStructuredData from '@/components/StructuredData/EntityStructuredData' +import StructuredData from '@/components/StructuredData' +import { AI_CRAWLER_USER_AGENTS } from '@/lib/seo/ai-crawlers' + +// The public site's machine-readable surfaces — sitemap, robots, JSON-LD — are +// the ones nobody looks at until a search engine has already acted on them. +// Every assertion here pins a decision that was made deliberately and would be +// silently reversible otherwise (a fake lastmod, a stray /mobile-demo entry, a +// Review node nobody can substantiate). + +const PRODUCTION_ORIGIN = 'https://useupup.com' + +/** Every `application/ld+json` payload in a rendered markup string. */ +function parseJsonLdBlocks(markup: string): unknown[] { + const blocks = [ + ...markup.matchAll( + /]*type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/g, + ), + ] + return blocks.map(match => JSON.parse(match[1]) as unknown) +} + +/** Flattens `@graph` containers so nodes can be looked up by `@type`. */ +function flattenGraph(documents: unknown[]): Record[] { + const nodes: Record[] = [] + for (const doc of documents) { + const record = doc as Record + const graph = record['@graph'] + if (Array.isArray(graph)) + nodes.push(...(graph as Record[])) + else nodes.push(record) + } + return nodes +} + +function nodeOfType( + nodes: Record[], + type: string, +): Record { + const found = nodes.find(node => node['@type'] === type) + expect(found, `no ${type} node in the rendered JSON-LD`).toBeDefined() + return found as Record +} + +describe('sitemap enumerates only canonical, indexable page URLs', () => { + const entries = sitemap() + + it('lists the homepage, six framework pages, support and privacy, and all 64 docs pages', () => { + // 1 home + 6 frameworks + support + privacy + 64 fumadocs pages. The + // docs count is independently pinned by docs-source.test.ts, so a page + // added to content/docs updates both or neither. + expect(entries).toHaveLength(1 + 6 + 2 + 64) + }) + + it('points every entry at the production origin with the trailing slash the site actually serves', () => { + for (const entry of entries) { + expect(entry.url.startsWith(`${PRODUCTION_ORIGIN}/`)).toBe(true) + expect(entry.url.endsWith('/')).toBe(true) + } + }) + + it('omits the demo harness, the API routes, and the agent-only text surfaces', () => { + // /mobile-demo and /api are robots-disallowed; /docs-md and /llms*.txt + // are alternate representations of pages already listed here, not pages. + const excluded = ['/mobile-demo', '/api/', '/docs-md', '/llms'] + for (const entry of entries) { + for (const fragment of excluded) { + expect( + entry.url.includes(fragment), + `${entry.url} must not contain ${fragment}`, + ).toBe(false) + } + } + }) + + it('stamps no lastModified date on any entry', () => { + // A build-time `new Date()` on all 73 URLs claimed the whole site + // changed on every deploy; Google's response to an uncorroborated + // lastmod is to ignore the field site-wide. No real per-page date + // exists, so the field stays absent rather than invented. + for (const entry of entries) { + expect(entry.lastModified, `${entry.url} lastModified`).toBe( + undefined, + ) + } + }) +}) + +describe('robots.txt grants crawl access per host and names the AI agents', () => { + afterEach(() => { + vi.unstubAllEnvs() + vi.resetModules() + }) + + it('allows the whole site to the wildcard agent except the API and demo harness', () => { + const rules = [robots().rules].flat() + const wildcard = rules.find(rule => rule.userAgent === '*') + expect(wildcard?.allow).toBe('/') + expect(wildcard?.disallow).toEqual(['/api/', '/mobile-demo/']) + }) + + it('carries a second rule listing exactly the AI crawler allow-list', () => { + const rules = [robots().rules].flat() + const agentRule = rules.find(rule => Array.isArray(rule.userAgent)) + expect(agentRule?.userAgent).toEqual([...AI_CRAWLER_USER_AGENTS]) + expect(agentRule?.allow).toBe('/') + // The named agents must never get a laxer disallow set than `*`. + expect(agentRule?.disallow).toEqual(['/api/', '/mobile-demo/']) + }) + + it('advertises the production sitemap URL', () => { + expect(robots().sitemap).toBe(`${PRODUCTION_ORIGIN}/sitemap.xml`) + }) + + it('blanket-disallows crawling when the deployment is not the production site', async () => { + // clientEnv is parsed once at module load, so the stub has to land + // before a FRESH import of the robots route and the site-url helper + // underneath it. + vi.stubEnv('NEXT_PUBLIC_BASE_URL', 'https://dev.useupup.com') + vi.resetModules() + const devRobots = (await import('@/app/robots')).default + const result = devRobots() + expect(result.rules).toEqual([{ userAgent: '*', disallow: '/' }]) + expect(result.sitemap).toBe('https://dev.useupup.com/sitemap.xml') + }) +}) + +describe('entity JSON-LD ties the brand to one referenced organization', () => { + const entityNodes = flattenGraph( + parseJsonLdBlocks( + renderToStaticMarkup(createElement(EntityStructuredData)), + ), + ) + + it('describes the Organization with a stable id, alternate names, and verified profiles', () => { + const org = nodeOfType(entityNodes, 'Organization') + expect(org['@id']).toBe(`${PRODUCTION_ORIGIN}/#organization`) + expect(org.name).toBe('upup') + expect(org.alternateName).toContain('useupup') + expect(org.alternateName).toContain('@useupup') + const sameAs = org.sameAs as string[] + expect(sameAs.length).toBeGreaterThanOrEqual(2) + for (const profile of sameAs) + expect(profile.startsWith('https://')).toBe(true) + }) + + it('names Devino as the parent organization', () => { + const org = nodeOfType(entityNodes, 'Organization') + expect(org.parentOrganization).toEqual({ + '@type': 'Organization', + name: 'Devino', + url: 'https://devino.ca/', + }) + }) + + it('makes the WebSite publisher reference the organization node by id', () => { + const site = nodeOfType(entityNodes, 'WebSite') + expect(site['@id']).toBe(`${PRODUCTION_ORIGIN}/#website`) + expect(site.publisher).toEqual({ + '@id': `${PRODUCTION_ORIGIN}/#organization`, + }) + }) + + it('makes the page-level SoftwareApplication publisher reference the same organization node', () => { + const pageNodes = flattenGraph( + parseJsonLdBlocks( + renderToStaticMarkup(createElement(StructuredData)), + ), + ) + const app = nodeOfType(pageNodes, 'SoftwareApplication') + expect(app['@id']).toBe(`${PRODUCTION_ORIGIN}/#software`) + expect(app.publisher).toEqual({ + '@id': `${PRODUCTION_ORIGIN}/#organization`, + }) + expect(app.author).toEqual({ + '@id': `${PRODUCTION_ORIGIN}/#organization`, + }) + }) + + it('emits no AggregateRating or Review anywhere in the rendered markup', () => { + // We have no first-party review corpus. Rating markup we cannot + // substantiate is a manual-action risk, so its absence is a pin, not + // an omission — this fails the moment someone adds one. + const markup = + renderToStaticMarkup(createElement(EntityStructuredData)) + + renderToStaticMarkup(createElement(StructuredData)) + expect(markup).not.toContain('AggregateRating') + expect(markup).not.toContain('"Review"') + }) +}) + +describe('AI crawler allow-list holds the exact agreed agent names', () => { + it('pins all eighteen user-agent tokens in their published spelling', () => { + // Spelling is load-bearing: robots.txt user-agent matching is on these + // literal tokens, so a "tidied" name silently drops that agent's group. + expect([...AI_CRAWLER_USER_AGENTS]).toEqual([ + 'GPTBot', + 'OAI-SearchBot', + 'ChatGPT-User', + 'ClaudeBot', + 'Claude-User', + 'Claude-SearchBot', + 'anthropic-ai', + 'PerplexityBot', + 'Perplexity-User', + 'Google-Extended', + 'Googlebot', + 'Bingbot', + 'Applebot', + 'Applebot-Extended', + 'CCBot', + 'Amazonbot', + 'Bytespider', + 'meta-externalagent', + ]) + }) +}) diff --git a/apps/landing/src/app/docs-md/[[...slug]]/route.ts b/apps/landing/src/app/docs-md/[[...slug]]/route.ts index 9bdfbcf60..a966506fc 100644 --- a/apps/landing/src/app/docs-md/[[...slug]]/route.ts +++ b/apps/landing/src/app/docs-md/[[...slug]]/route.ts @@ -1,4 +1,5 @@ import { loadPages } from '@/lib/docs/llms' +import { canonicalUrl } from '@/lib/site-url' // One raw-markdown endpoint per docs page, consumed by the "Copy page" button. // Frozen at build time like the llms.txt routes — a docs edit appears only @@ -28,7 +29,16 @@ export async function GET( // lives in frontmatter and is rendered as the

on the article page), so // prepend it — mirrors buildLlmsFull's per-page shape. const markdown = `# ${page.title}\n\n${page.body}\n` + // RFC 8288 canonical link. This route serves the same content as the HTML + // docs page under a second URL, so without it a crawler that discovers the + // twin has two competing originals. A header (rather than a noindex) is the + // right tool here: agents are welcome to fetch and quote these bytes, they + // just must not treat the twin as a page in its own right. + const canonical = canonicalUrl(target ? `docs/${target}` : 'docs') return new Response(markdown, { - headers: { 'content-type': 'text/markdown; charset=utf-8' }, + headers: { + 'content-type': 'text/markdown; charset=utf-8', + link: `<${canonical}>; rel="canonical"`, + }, }) } diff --git a/apps/landing/src/app/docs/[[...slug]]/page.tsx b/apps/landing/src/app/docs/[[...slug]]/page.tsx index 44cb304e2..0e44d0458 100644 --- a/apps/landing/src/app/docs/[[...slug]]/page.tsx +++ b/apps/landing/src/app/docs/[[...slug]]/page.tsx @@ -8,6 +8,7 @@ import { DocsToc } from '@/components/docs/DocsToc' import { DocsHome } from '@/components/docs/DocsHome' import { DocsPageNav } from '@/components/docs/DocsPageNav' import { DocsCopyPage } from '@/components/docs/DocsCopyPage' +import { DocsStructuredData } from '@/components/docs/DocsStructuredData' import { canonicalUrl, siteUrl } from '@/lib/site-url' // content/docs is edited on the canonical master branch on GitHub. @@ -33,11 +34,22 @@ export async function generateMetadata(props: { const description = page.data.description const url = canonicalUrl(slug?.length ? `docs/${slug.join('/')}` : 'docs') const image = `${siteUrl()}/img/social-card.png` + // The raw-markdown twin is an alternate REPRESENTATION of this page, not a + // second page. Declaring it renders , which is how an agent finds the token-cheap copy + // without us needing it to be indexed in its own right (the route answers + // with a `Link: …; rel="canonical"` header pointing back here). + const markdownUrl = `${siteUrl()}/docs-md/${ + slug?.length ? `${slug.join('/')}/` : '' + }` return { title, description, - alternates: { canonical: url }, + alternates: { + canonical: url, + types: { 'text/markdown': markdownUrl }, + }, openGraph: { title, description, @@ -80,8 +92,27 @@ export default async function DocsPage(props: { with the copy button, so no extra bottom margin here. */}
- + {/* The Copy button fetches the twin from JS; this anchor is + the crawlable path to the same bytes — a link an agent + (or a person who wants the markdown) can actually + follow. Both point at the slashed canonical URL. */} +
+ {/* prose-code:before/after content-none: the typography plugin's default renders literal backtick glyphs around inline code; the chip styling replaces them, scoped via diff --git a/apps/landing/src/app/layout.tsx b/apps/landing/src/app/layout.tsx index bd1bfaac8..ca2498dd5 100644 --- a/apps/landing/src/app/layout.tsx +++ b/apps/landing/src/app/layout.tsx @@ -10,6 +10,7 @@ import { Providers } from '@/components/providers' import { PostHogProvider } from '@/components/posthog-provider' import Navbar from '@/components/Navbar' import Footer from '@/components/Footer' +import EntityStructuredData from '@/components/StructuredData/EntityStructuredData' const geistSans = Geist({ variable: '--font-geist-sans', @@ -152,6 +153,11 @@ export default function RootLayout({ })(); `} + {/* Organization + WebSite JSON-LD. It lives in the ROOT + layout on purpose: the entity graph has to be on every + page — the 64 docs pages are the bulk of the indexable + surface and they mount no page-level . */} + {process.env.NODE_ENV === 'production' && ( <>