feat(scoring,crawler): skip noindex pages in per-page scoring, honour nofollow

.changeset/crawler-nofollow.md

@@ -0,0 +1,5 @@
+---
+"@advance-labs/crawler": patch
+---
+
+Honour `nofollow`: links on a page whose robots meta or `X-Robots-Tag` says `nofollow` (or `none`) are not followed, and `<a rel="nofollow">` links are skipped. The page itself is still fetched and recorded. Exports `pageForbidsFollow`.

.changeset/scoring-noindex-skip.md

@@ -0,0 +1,6 @@
+---
+"@advance-labs/scoring": minor
+"@advance-labs/types": minor
+---
+
+Stop scoring pages that opt out of indexing. Pages marked `noindex` (meta robots or `X-Robots-Tag`) are left out of per-page rule scoring (titles, descriptions, thin content, headings, Open Graph, structured data, ...) and listed in the new `AuditReport.excludedPages` (`reason: 'noindex'`); they still count in `pagesCrawled`. Rules that inspect indexability opt back in with the new `Rule.includesNoindexPages`: `tech.indexable` (now ignores a noindex page the sitemap deliberately leaves out, still flags the homepage, any listed noindex page, and every noindex page when there is no sitemap), `tech.canonical-present` and `tech.canonical-resolves`. If every page is noindex nothing is excluded.

packages/crawler/src/crawl.test.ts

@@ -233,6 +233,45 @@ describe('crawl', () => {
     expect(fetched.filter((u) => u === 'https://example.com/')).toHaveLength(1);
   });
 
+  it('fetches a noindex,nofollow page but does not follow its links', async () => {
+    const { fetcher, fetched } = fakeSite({
+      'https://example.com/': () => htmlResponse(HTML(['/login', '/about'])),
+      'https://example.com/login': () =>
+        htmlResponse(`<html><head><meta name="robots" content="noindex, nofollow"></head><body><a href="/behind-login">x</a></body></html>`),
+      'https://example.com/about': () => htmlResponse(HTML([])),
+      'https://example.com/behind-login': () => htmlResponse(HTML([])),
+    });
+    const result = await crawl('https://example.com/', { maxPages: 10, delay: instantDelay, fetcher });
+    expect(result.pages.map((p) => p.finalUrl)).toContain('https://example.com/login');
+    expect(fetched).not.toContain('https://example.com/behind-login');
+  });
+
+  it('follows the links of a noindex-only page', async () => {
+    const { fetcher, fetched } = fakeSite({
+      'https://example.com/': () =>
+        htmlResponse(`<html><head><meta name="robots" content="noindex"></head><body><a href="/next">x</a></body></html>`),
+      'https://example.com/next': () => htmlResponse(HTML([])),
+    });
+    await crawl('https://example.com/', { maxPages: 10, delay: instantDelay, fetcher });
+    expect(fetched).toContain('https://example.com/next');
+  });
+
+  it('honours X-Robots-Tag: nofollow and rel=nofollow links', async () => {
+    const { fetcher, fetched } = fakeSite({
+      'https://example.com/': () =>
+        htmlResponse(`<a href="/hdr">h</a><a rel="nofollow" href="/skip">s</a><a href="/ok">o</a>`),
+      'https://example.com/hdr': () =>
+        makeResponse({ status: 200, headers: { 'content-type': 'text/html', 'x-robots-tag': 'nofollow' }, body: HTML(['/hidden']) }),
+      'https://example.com/skip': () => htmlResponse(HTML([])),
+      'https://example.com/ok': () => htmlResponse(HTML([])),
+      'https://example.com/hidden': () => htmlResponse(HTML([])),
+    });
+    await crawl('https://example.com/', { maxPages: 10, delay: instantDelay, fetcher });
+    expect(fetched).toContain('https://example.com/ok');
+    expect(fetched).not.toContain('https://example.com/skip');
+    expect(fetched).not.toContain('https://example.com/hidden');
+  });
+
   it('crawls from where an apex -> www redirect lands', async () => {
     const { fetcher } = fakeSite({
       'https://example.com/': () => makeResponse({ status: 308, headers: { location: 'https://www.example.com/' } }),

packages/crawler/src/crawl.ts

@@ -12,7 +12,7 @@ import { fetchResource } from './fetch-resource.js';
 import { parseRobotsTxt, emptyRobotsTxt } from './robots.js';
 import { parseSitemapDocument } from './sitemap.js';
 import { detectSiteFiles } from './site-files.js';
-import { canonicalUrl, extractLinks, hostOf, sameOrigin, sameRegistrableSite } from './links.js';
+import { canonicalUrl, extractLinks, hostOf, pageForbidsFollow, sameOrigin, sameRegistrableSite } from './links.js';
 import { PerHostRateLimiter } from './rate-limit.js';
 import {
   DEFAULT_CONCURRENCY,
@@ -123,7 +123,7 @@ export async function crawl(requestedUrl: Url, opts: CrawlRuntimeOptions): Promi
       if (pages.length >= maxPages) break;
       pages.push(page);
       // Only follow links from successful HTML responses.
-      if (page.ok && isHtml(page.contentType) && page.body) {
+      if (page.ok && isHtml(page.contentType) && page.body && !pageForbidsFollow(page.headers, page.body)) {
         for (const link of extractLinks(page.body, page.finalUrl)) {
           enqueue(link, page.depth + 1, page.finalUrl);
         }

packages/crawler/src/index.ts

@@ -19,7 +19,7 @@ export type { SitemapKind, ParsedSitemap } from './sitemap.js';
 export { detectSiteFiles } from './site-files.js';
 export type { DetectSiteFilesOptions } from './site-files.js';
 
-export { extractLinks, canonicalUrl, sameOrigin, sameRegistrableSite, hostOf } from './links.js';
+export { extractLinks, canonicalUrl, pageForbidsFollow, sameOrigin, sameRegistrableSite, hostOf } from './links.js';
 export { PerHostRateLimiter } from './rate-limit.js';
 
 export { resolveFetcher, CrawlerError } from './fetcher.js';

packages/crawler/src/links.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect } from 'vitest';
-import { extractLinks, canonicalUrl, sameOrigin, sameRegistrableSite, hostOf } from './links.js';
+import { extractLinks, canonicalUrl, pageForbidsFollow, sameOrigin, sameRegistrableSite, hostOf } from './links.js';
 
 const BASE = 'https://example.com/blog/';
 
@@ -82,3 +82,41 @@ describe('canonicalUrl', () => {
     expect(canonicalUrl('not a url')).toBe('not a url');
   });
 });
+
+describe('rel=nofollow links', () => {
+  it('skips <a rel=nofollow> (any quoting, token list) and keeps everything else', () => {
+    const html = `
+      <a href="/keep">keep</a>
+      <a rel="nofollow" href="/a">a</a>
+      <a href="/b" rel='noopener nofollow'>b</a>
+      <a href="/c" rel=nofollow>c</a>
+      <a href="/d" rel="noopener">d</a>
+      <a href="/e" data-rel="nofollow">e</a>
+      <link rel="canonical" href="/canon">
+    `;
+    expect(extractLinks(html, 'https://example.com/')).toEqual([
+      'https://example.com/keep',
+      'https://example.com/d',
+      'https://example.com/e',
+      'https://example.com/canon',
+    ]);
+  });
+});
+
+describe('pageForbidsFollow', () => {
+  const meta = (c: string): string => `<head><meta name="robots" content="${c}"></head>`;
+  it('is true for nofollow or none in meta robots', () => {
+    expect(pageForbidsFollow({}, meta('noindex, nofollow'))).toBe(true);
+    expect(pageForbidsFollow({}, meta('none'))).toBe(true);
+  });
+  it('is false for noindex alone, index/follow, or no directive', () => {
+    expect(pageForbidsFollow({}, meta('noindex'))).toBe(false);
+    expect(pageForbidsFollow({}, meta('index, follow'))).toBe(false);
+    expect(pageForbidsFollow({}, '<html></html>')).toBe(false);
+  });
+  it('reads X-Robots-Tag, ignoring other agents', () => {
+    expect(pageForbidsFollow({ 'x-robots-tag': 'nofollow' }, '')).toBe(true);
+    expect(pageForbidsFollow({ 'x-robots-tag': 'googlebot: nofollow' }, '')).toBe(true);
+    expect(pageForbidsFollow({ 'x-robots-tag': 'otherbot: nofollow' }, '')).toBe(false);
+  });
+});

packages/crawler/src/links.ts

@@ -25,6 +25,7 @@ export function extractLinks(html: string, baseUrl: Url): Url[] {
     if (rawHref === undefined) continue;
     const href = rawHref.trim();
     if (href.length === 0 || href.startsWith('#')) continue;
+    if (isNofollowTag(html, match.index ?? 0)) continue;
 
     const resolved = resolveHref(href, baseUrl);
     if (!resolved) continue;
@@ -36,6 +37,48 @@ export function extractLinks(html: string, baseUrl: Url): Url[] {
   return out;
 }
 
+const REL_RE = /(?<![\w-])rel\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+))/i;
+
+/** Whether the tag containing `index` is an `<a>` carrying `rel="nofollow"`. */
+function isNofollowTag(html: string, index: number): boolean {
+  const start = html.lastIndexOf('<', index);
+  const end = html.indexOf('>', index);
+  if (start === -1 || end === -1) return false;
+  const tag = html.slice(start, end + 1);
+  if (!/^<a[\s>]/i.test(tag)) return false;
+  const rel = REL_RE.exec(tag);
+  const value = rel?.[1] ?? rel?.[2] ?? rel?.[3] ?? '';
+  return value.toLowerCase().split(/\s+/).includes('nofollow');
+}
+
+const META_ROBOTS_RE = /<meta\b[^>]*\bname\s*=\s*["']?robots["']?[^>]*>/gi;
+const CONTENT_RE = /\bcontent\s*=\s*(?:"([^"]*)"|'([^']*)')/i;
+
+/**
+ * Whether a page asks crawlers not to follow its links: `nofollow` or `none` in a robots meta tag
+ * or an X-Robots-Tag header. A `noindex`-only page still has its links followed.
+ */
+export function pageForbidsFollow(headers: Record<string, string>, html: string | undefined): boolean {
+  const values: string[] = [];
+  const header = headers['x-robots-tag'];
+  if (header) {
+    // "googlebot: nofollow" is aimed at one agent; only unprefixed and search-engine ones count.
+    for (const part of header.toLowerCase().split(',')) {
+      const m = /^\s*([a-z0-9._-]+)\s*:\s*(.+)$/.exec(part);
+      if (!m) values.push(part);
+      else if (m[1] === 'googlebot' || m[1] === 'bingbot') values.push(m[2] ?? '');
+    }
+  }
+  for (const tag of (html ?? '').match(META_ROBOTS_RE) ?? []) {
+    const c = CONTENT_RE.exec(tag);
+    values.push((c?.[1] ?? c?.[2] ?? '').toLowerCase());
+  }
+  return values
+    .flatMap((v) => v.split(','))
+    .map((t) => t.trim())
+    .some((t) => t === 'nofollow' || t === 'none');
+}
+
 /**
  * Canonical frontier key for a URL: parsed and re-serialised (so an empty path becomes `/`, the
  * host is lower-cased and default ports dropped) with the fragment stripped. The query string is

packages/scoring/src/audit.ts

@@ -9,6 +9,7 @@
  */
 import type { AuditReport, Finding, FindingSeverity, Score, ScoringContext } from '@advance-labs/types';
 import { runRules } from './engine.js';
+import { withoutNoindexPages } from './context-utils.js';
 import { technicalSeoRules } from './technical-seo-rules.js';
 import { aeoRules } from './aeo-rules.js';
 import { generateTemplates } from './templates.js';
@@ -66,12 +67,14 @@ export async function buildAuditReport(
   const allFindings = categories.flatMap((c) => c.findings);
   const topFixes = prioritizeFixes(allFindings);
   const templates = generateTemplates(ctx);
+  const { excluded } = withoutNoindexPages(ctx);
 
   return {
     url: ctx.crawl.rootUrl,
     generatedAt: new Date().toISOString(),
     score,
     pagesCrawled: ctx.crawl.pageCount,
+    ...(excluded.length > 0 ? { excludedPages: excluded } : {}),
     topFixes,
     templates,
     meta: {

packages/scoring/src/context-utils.ts

@@ -148,10 +148,70 @@ export function pagesFailing(ctx: ScoringContext, pred: (page: ParsedHtml) => bo
   return out;
 }
 
+/** Directive tokens of a robots meta / X-Robots-Tag value, lower-cased. `none` means noindex+nofollow. */
+function robotsTokens(value: string | undefined, agentOk: (agent: string) => boolean): string[] {
+  const out: string[] = [];
+  for (const raw of (value ?? '').toLowerCase().split(',')) {
+    // X-Robots-Tag allows a leading user-agent: "googlebot: noindex". Only count agents that
+    // are search engines; "otherbot: noindex" says nothing about us.
+    const m = /^\s*([a-z0-9._-]+)\s*:\s*(.+)$/.exec(raw);
+    if (m && m[1] !== undefined && m[2] !== undefined) {
+      if (agentOk(m[1])) out.push(m[2].trim());
+    } else if (raw.trim()) out.push(raw.trim());
+  }
+  return out.flatMap((t) => (t === 'none' ? ['noindex', 'nofollow'] : [t]));
+}
+const isSearchAgent = (a: string): boolean => a === 'googlebot' || a === 'bingbot';
+
 /** Whether a robots `<meta>` directive on a page marks it noindex. */
 export function isNoindex(page: ParsedHtml): boolean {
-  const robots = page.meta.robots?.toLowerCase() ?? '';
-  return robots.includes('noindex');
+  return robotsTokens(page.meta.robots, () => true).includes('noindex');
+}
+
+/** Whether an `X-Robots-Tag` response header marks the page noindex. */
+export function isNoindexHeader(headers: Record<string, string> | undefined): boolean {
+  return robotsTokens(headers?.['x-robots-tag'], isSearchAgent).includes('noindex');
+}
+
+/** Normalized URLs of crawled pages that are noindex by meta robots or X-Robots-Tag. */
+export function noindexUrls(ctx: ScoringContext): Set<string> {
+  const out = new Set<string>();
+  for (const p of ctx.pages) if (isNoindex(p)) out.add(normalizeUrl(p.url));
+  for (const c of ctx.crawl.pages) {
+    if (isNoindexHeader(c.headers)) {
+      out.add(normalizeUrl(c.url));
+      out.add(normalizeUrl(c.finalUrl));
+    }
+  }
+  return out;
+}
+
+/**
+ * The context rules are scored against: parsed pages that opted out of indexing are removed
+ * (and, when `structuredData` lines up one-to-one with `pages`, their structured data too), and
+ * reported in `excluded`. `crawl` is left whole: robots/sitemap/redirect/broken-page rules are
+ * about the site and the crawl, not about indexed pages. If EVERY page is noindex the full
+ * context is returned untouched, since there is nothing left to score.
+ */
+export function withoutNoindexPages(ctx: ScoringContext): {
+  ctx: ScoringContext;
+  excluded: Array<{ url: Url; reason: 'noindex' }>;
+} {
+  const noindex = noindexUrls(ctx);
+  const keep = ctx.pages.map((p) => !noindex.has(normalizeUrl(p.url)));
+  const excluded = ctx.pages
+    .filter((_, i) => !keep[i])
+    .map((p) => ({ url: p.url, reason: 'noindex' as const }));
+  if (excluded.length === 0 || excluded.length === ctx.pages.length) return { ctx, excluded: [] };
+  const aligned = ctx.structuredData.length === ctx.pages.length;
+  return {
+    ctx: {
+      ...ctx,
+      pages: ctx.pages.filter((_, i) => keep[i]),
+      structuredData: aligned ? ctx.structuredData.filter((_, i) => keep[i]) : ctx.structuredData,
+    },
+    excluded,
+  };
 }
 
 /**

packages/scoring/src/engine.ts

@@ -22,6 +22,7 @@ import type {
   ScoringContext,
 } from '@advance-labs/types';
 import { clampScore, scoreToGrade } from './grade.js';
+import { withoutNoindexPages } from './context-utils.js';
 
 /** Human-readable labels for each category key, used in `ScoreCategory.label`. */
 const CATEGORY_LABELS: Record<ScoreCategoryKey, string> = {
@@ -118,14 +119,16 @@ export async function runRules(
 ): Promise<{ categories: ScoreCategory[]; score: Score }> {
   const byCategory = new Map<ScoreCategoryKey, Finding[]>();
   const order: ScoreCategoryKey[] = [];
+  const { ctx: scored } = withoutNoindexPages(ctx);
 
   for (const rule of rules) {
+    const view = rule.includesNoindexPages ? ctx : scored;
     // ADV-175: a rule that does not apply to this page contributes nothing at all — no
     // finding and no weight — rather than a pass that overstates coverage or a fail that
     // manufactures work.
-    if (rule.appliesTo && !rule.appliesTo(ctx)) continue;
+    if (rule.appliesTo && !rule.appliesTo(view)) continue;
 
-    const outcome = await evaluateRule(rule, ctx);
+    const outcome = await evaluateRule(rule, view);
     const finding = toFinding(rule, outcome);
     let bucket = byCategory.get(rule.category);
     if (!bucket) {

packages/scoring/src/index.ts

@@ -23,4 +23,4 @@ export type { BuildAuditReportOptions } from './audit.js';
 export { generateTemplates } from './templates.js';
 
 // Context helpers — useful for apps building their own ad-hoc rules.
-export { KEY_AI_BOTS } from './context-utils.js';
+export { KEY_AI_BOTS, withoutNoindexPages } from './context-utils.js';