docs: excludes comparison docs from search indexing (#891)

davemooreuws · web-flow · commit f9889ece3603 · 2025-09-17T15:58:54.000+10:00
diff --git a/docs/contentlayer.config.ts b/docs/contentlayer.config.ts
@@ -39,6 +39,10 @@ const baseFields: FieldDefs = {
     type: 'string',
     description: 'The canonical url of the doc, if different from the url',
   },
+  noindex: {
+    type: 'boolean',
+    description: 'Prevent search engines from indexing this page',
+  },
 }
 
 const computedFields: ComputedFields = {
diff --git a/docs/docs/misc/comparison/ampt.mdx b/docs/docs/misc/comparison/ampt.mdx
@@ -1,5 +1,6 @@
 ---
 description: 'Comparison to Ampt'
+noindex: true
 ---
 
 # Ampt vs. Nitric
diff --git a/docs/docs/misc/comparison/aws-cdk.mdx b/docs/docs/misc/comparison/aws-cdk.mdx
@@ -1,5 +1,6 @@
 ---
 description: 'Comparison to AWS CDK'
+noindex: true
 ---
 
 # AWS CDK vs. Nitric
diff --git a/docs/docs/misc/comparison/aws-sam.mdx b/docs/docs/misc/comparison/aws-sam.mdx
@@ -1,5 +1,6 @@
 ---
 description: 'Comparison to AWS Serverless Application Model (SAM)'
+noindex: true
 ---
 
 # AWS SAM vs. Nitric
diff --git a/docs/docs/misc/comparison/bicep.mdx b/docs/docs/misc/comparison/bicep.mdx
@@ -1,5 +1,6 @@
 ---
 description: 'Comparison to Bicep'
+noindex: true
 ---
 
 # Bicep vs. Nitric
diff --git a/docs/docs/misc/comparison/encore.mdx b/docs/docs/misc/comparison/encore.mdx
@@ -1,5 +1,6 @@
 ---
 description: 'Comparison to Encore'
+noindex: true
 ---
 
 # Encore vs. Nitric
diff --git a/docs/docs/misc/comparison/gcp-deployment-manager.mdx b/docs/docs/misc/comparison/gcp-deployment-manager.mdx
@@ -1,5 +1,6 @@
 ---
 description: 'Comparison to Google Cloud Deployment Manager'
+noindex: true
 ---
 
 # Google Cloud Deployment Manager vs. Nitric
diff --git a/docs/docs/misc/comparison/pulumi.mdx b/docs/docs/misc/comparison/pulumi.mdx
@@ -1,5 +1,6 @@
 ---
 description: 'Comparison to Pulumi'
+noindex: true
 ---
 
 # Pulumi vs. Nitric
diff --git a/docs/docs/misc/comparison/sst.mdx b/docs/docs/misc/comparison/sst.mdx
@@ -1,5 +1,6 @@
 ---
 description: 'Comparison to SST'
+noindex: true
 ---
 
 # SST vs. Nitric
diff --git a/docs/docs/misc/comparison/terraform.mdx b/docs/docs/misc/comparison/terraform.mdx
@@ -1,5 +1,6 @@
 ---
 description: 'Comparison to Terraform'
+noindex: true
 ---
 
 # Terraform vs. Nitric
diff --git a/docs/src/app/(sitemaps)/sitemap-0.xml/route.ts b/docs/src/app/(sitemaps)/sitemap-0.xml/route.ts
@@ -22,12 +22,14 @@ export async function GET() {
     priority: 0.7,
   }))
 
-  const docPages: SitemapItem[] = allDocuments.map((page) => ({
-    loc: page.slug === '' ? URL : `${URL}/${page.slug}`,
-    lastmod: new Date(page.lastModified).toISOString(),
-    changefreq: 'daily',
-    priority: 0.7,
-  }))
+  const docPages: SitemapItem[] = allDocuments
+    .filter((page) => !page.slug.startsWith('misc/comparison/'))
+    .map((page) => ({
+      loc: page.slug === '' ? URL : `${URL}/${page.slug}`,
+      lastmod: new Date(page.lastModified).toISOString(),
+      changefreq: 'daily',
+      priority: 0.7,
+    }))
 
   const allPagesSorted = [...pages, ...docPages].sort((a, b) =>
     a.loc < b.loc ? -1 : 1,
diff --git a/docs/src/app/[[...slug]]/page.tsx b/docs/src/app/[[...slug]]/page.tsx
@@ -72,6 +72,12 @@ export async function generateMetadata({
     alternates: {
       canonical: doc.canonical_url ? doc.canonical_url : url,
     },
+    ...(doc.noindex && {
+      robots: {
+        index: false,
+        follow: false,
+      },
+    }),
   }
 }
 
diff --git a/docs/src/app/robots.ts b/docs/src/app/robots.ts
@@ -13,10 +13,13 @@ export default function robots(): MetadataRoute.Robots {
 
   // production robots.txt
   return {
-    rules: {
-      userAgent: '*',
-      allow: '/',
-    },
+    rules: [
+      {
+        userAgent: '*',
+        allow: '/',
+        disallow: '/misc/comparison/',
+      },
+    ],
     host: 'https://nitric.io/docs',
     sitemap: 'https://nitric.io/docs/sitemap.xml',
   }
