diff --git a/.changeset/crawler-nofollow.md b/.changeset/crawler-nofollow.md new file mode 100644 index 0000000..e5c42ad --- /dev/null +++ b/.changeset/crawler-nofollow.md @@ -0,0 +1,5 @@ +--- +"@advance-labs/crawler": patch +--- + +Honour `nofollow`: links on a page whose robots meta or `X-Robots-Tag` says `nofollow` (or `none`) are not followed, and `` links are skipped. The page itself is still fetched and recorded. Exports `pageForbidsFollow`. diff --git a/.changeset/scoring-noindex-skip.md b/.changeset/scoring-noindex-skip.md new file mode 100644 index 0000000..974eeb9 --- /dev/null +++ b/.changeset/scoring-noindex-skip.md @@ -0,0 +1,6 @@ +--- +"@advance-labs/scoring": minor +"@advance-labs/types": minor +--- + +Stop scoring pages that opt out of indexing. Pages marked `noindex` (meta robots or `X-Robots-Tag`) are left out of per-page rule scoring (titles, descriptions, thin content, headings, Open Graph, structured data, ...) and listed in the new `AuditReport.excludedPages` (`reason: 'noindex'`); they still count in `pagesCrawled`. Rules that inspect indexability opt back in with the new `Rule.includesNoindexPages`: `tech.indexable` (now ignores a noindex page the sitemap deliberately leaves out, still flags the homepage, any listed noindex page, and every noindex page when there is no sitemap), `tech.canonical-present` and `tech.canonical-resolves`. If every page is noindex nothing is excluded. diff --git a/packages/crawler/src/crawl.test.ts b/packages/crawler/src/crawl.test.ts index 271a492..0de8acc 100644 --- a/packages/crawler/src/crawl.test.ts +++ b/packages/crawler/src/crawl.test.ts @@ -233,6 +233,45 @@ describe('crawl', () => { expect(fetched.filter((u) => u === 'https://example.com/')).toHaveLength(1); }); + it('fetches a noindex,nofollow page but does not follow its links', async () => { + const { fetcher, fetched } = fakeSite({ + 'https://example.com/': () => htmlResponse(HTML(['/login', '/about'])), + 'https://example.com/login': () => + htmlResponse(`x`), + 'https://example.com/about': () => htmlResponse(HTML([])), + 'https://example.com/behind-login': () => htmlResponse(HTML([])), + }); + const result = await crawl('https://example.com/', { maxPages: 10, delay: instantDelay, fetcher }); + expect(result.pages.map((p) => p.finalUrl)).toContain('https://example.com/login'); + expect(fetched).not.toContain('https://example.com/behind-login'); + }); + + it('follows the links of a noindex-only page', async () => { + const { fetcher, fetched } = fakeSite({ + 'https://example.com/': () => + htmlResponse(`x`), + 'https://example.com/next': () => htmlResponse(HTML([])), + }); + await crawl('https://example.com/', { maxPages: 10, delay: instantDelay, fetcher }); + expect(fetched).toContain('https://example.com/next'); + }); + + it('honours X-Robots-Tag: nofollow and rel=nofollow links', async () => { + const { fetcher, fetched } = fakeSite({ + 'https://example.com/': () => + htmlResponse(`hso`), + 'https://example.com/hdr': () => + makeResponse({ status: 200, headers: { 'content-type': 'text/html', 'x-robots-tag': 'nofollow' }, body: HTML(['/hidden']) }), + 'https://example.com/skip': () => htmlResponse(HTML([])), + 'https://example.com/ok': () => htmlResponse(HTML([])), + 'https://example.com/hidden': () => htmlResponse(HTML([])), + }); + await crawl('https://example.com/', { maxPages: 10, delay: instantDelay, fetcher }); + expect(fetched).toContain('https://example.com/ok'); + expect(fetched).not.toContain('https://example.com/skip'); + expect(fetched).not.toContain('https://example.com/hidden'); + }); + it('crawls from where an apex -> www redirect lands', async () => { const { fetcher } = fakeSite({ 'https://example.com/': () => makeResponse({ status: 308, headers: { location: 'https://www.example.com/' } }), diff --git a/packages/crawler/src/crawl.ts b/packages/crawler/src/crawl.ts index bcfebcf..0ff17ba 100644 --- a/packages/crawler/src/crawl.ts +++ b/packages/crawler/src/crawl.ts @@ -12,7 +12,7 @@ import { fetchResource } from './fetch-resource.js'; import { parseRobotsTxt, emptyRobotsTxt } from './robots.js'; import { parseSitemapDocument } from './sitemap.js'; import { detectSiteFiles } from './site-files.js'; -import { canonicalUrl, extractLinks, hostOf, sameOrigin, sameRegistrableSite } from './links.js'; +import { canonicalUrl, extractLinks, hostOf, pageForbidsFollow, sameOrigin, sameRegistrableSite } from './links.js'; import { PerHostRateLimiter } from './rate-limit.js'; import { DEFAULT_CONCURRENCY, @@ -123,7 +123,7 @@ export async function crawl(requestedUrl: Url, opts: CrawlRuntimeOptions): Promi if (pages.length >= maxPages) break; pages.push(page); // Only follow links from successful HTML responses. - if (page.ok && isHtml(page.contentType) && page.body) { + if (page.ok && isHtml(page.contentType) && page.body && !pageForbidsFollow(page.headers, page.body)) { for (const link of extractLinks(page.body, page.finalUrl)) { enqueue(link, page.depth + 1, page.finalUrl); } diff --git a/packages/crawler/src/index.ts b/packages/crawler/src/index.ts index 13b7176..c552b7b 100644 --- a/packages/crawler/src/index.ts +++ b/packages/crawler/src/index.ts @@ -19,7 +19,7 @@ export type { SitemapKind, ParsedSitemap } from './sitemap.js'; export { detectSiteFiles } from './site-files.js'; export type { DetectSiteFilesOptions } from './site-files.js'; -export { extractLinks, canonicalUrl, sameOrigin, sameRegistrableSite, hostOf } from './links.js'; +export { extractLinks, canonicalUrl, pageForbidsFollow, sameOrigin, sameRegistrableSite, hostOf } from './links.js'; export { PerHostRateLimiter } from './rate-limit.js'; export { resolveFetcher, CrawlerError } from './fetcher.js'; diff --git a/packages/crawler/src/links.test.ts b/packages/crawler/src/links.test.ts index 098d9f6..7e6a3ea 100644 --- a/packages/crawler/src/links.test.ts +++ b/packages/crawler/src/links.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { extractLinks, canonicalUrl, sameOrigin, sameRegistrableSite, hostOf } from './links.js'; +import { extractLinks, canonicalUrl, pageForbidsFollow, sameOrigin, sameRegistrableSite, hostOf } from './links.js'; const BASE = 'https://example.com/blog/'; @@ -82,3 +82,41 @@ describe('canonicalUrl', () => { expect(canonicalUrl('not a url')).toBe('not a url'); }); }); + +describe('rel=nofollow links', () => { + it('skips (any quoting, token list) and keeps everything else', () => { + const html = ` + keep + a + b + c + d + e + + `; + expect(extractLinks(html, 'https://example.com/')).toEqual([ + 'https://example.com/keep', + 'https://example.com/d', + 'https://example.com/e', + 'https://example.com/canon', + ]); + }); +}); + +describe('pageForbidsFollow', () => { + const meta = (c: string): string => ``; + it('is true for nofollow or none in meta robots', () => { + expect(pageForbidsFollow({}, meta('noindex, nofollow'))).toBe(true); + expect(pageForbidsFollow({}, meta('none'))).toBe(true); + }); + it('is false for noindex alone, index/follow, or no directive', () => { + expect(pageForbidsFollow({}, meta('noindex'))).toBe(false); + expect(pageForbidsFollow({}, meta('index, follow'))).toBe(false); + expect(pageForbidsFollow({}, '')).toBe(false); + }); + it('reads X-Robots-Tag, ignoring other agents', () => { + expect(pageForbidsFollow({ 'x-robots-tag': 'nofollow' }, '')).toBe(true); + expect(pageForbidsFollow({ 'x-robots-tag': 'googlebot: nofollow' }, '')).toBe(true); + expect(pageForbidsFollow({ 'x-robots-tag': 'otherbot: nofollow' }, '')).toBe(false); + }); +}); diff --git a/packages/crawler/src/links.ts b/packages/crawler/src/links.ts index 95ed62d..7ad0636 100644 --- a/packages/crawler/src/links.ts +++ b/packages/crawler/src/links.ts @@ -25,6 +25,7 @@ export function extractLinks(html: string, baseUrl: Url): Url[] { if (rawHref === undefined) continue; const href = rawHref.trim(); if (href.length === 0 || href.startsWith('#')) continue; + if (isNofollowTag(html, match.index ?? 0)) continue; const resolved = resolveHref(href, baseUrl); if (!resolved) continue; @@ -36,6 +37,48 @@ export function extractLinks(html: string, baseUrl: Url): Url[] { return out; } +const REL_RE = /(?]+))/i; + +/** Whether the tag containing `index` is an `` carrying `rel="nofollow"`. */ +function isNofollowTag(html: string, index: number): boolean { + const start = html.lastIndexOf('<', index); + const end = html.indexOf('>', index); + if (start === -1 || end === -1) return false; + const tag = html.slice(start, end + 1); + if (!/^]/i.test(tag)) return false; + const rel = REL_RE.exec(tag); + const value = rel?.[1] ?? rel?.[2] ?? rel?.[3] ?? ''; + return value.toLowerCase().split(/\s+/).includes('nofollow'); +} + +const META_ROBOTS_RE = /]*\bname\s*=\s*["']?robots["']?[^>]*>/gi; +const CONTENT_RE = /\bcontent\s*=\s*(?:"([^"]*)"|'([^']*)')/i; + +/** + * Whether a page asks crawlers not to follow its links: `nofollow` or `none` in a robots meta tag + * or an X-Robots-Tag header. A `noindex`-only page still has its links followed. + */ +export function pageForbidsFollow(headers: Record, html: string | undefined): boolean { + const values: string[] = []; + const header = headers['x-robots-tag']; + if (header) { + // "googlebot: nofollow" is aimed at one agent; only unprefixed and search-engine ones count. + for (const part of header.toLowerCase().split(',')) { + const m = /^\s*([a-z0-9._-]+)\s*:\s*(.+)$/.exec(part); + if (!m) values.push(part); + else if (m[1] === 'googlebot' || m[1] === 'bingbot') values.push(m[2] ?? ''); + } + } + for (const tag of (html ?? '').match(META_ROBOTS_RE) ?? []) { + const c = CONTENT_RE.exec(tag); + values.push((c?.[1] ?? c?.[2] ?? '').toLowerCase()); + } + return values + .flatMap((v) => v.split(',')) + .map((t) => t.trim()) + .some((t) => t === 'nofollow' || t === 'none'); +} + /** * Canonical frontier key for a URL: parsed and re-serialised (so an empty path becomes `/`, the * host is lower-cased and default ports dropped) with the fragment stripped. The query string is diff --git a/packages/scoring/src/audit.ts b/packages/scoring/src/audit.ts index 612abd2..637d133 100644 --- a/packages/scoring/src/audit.ts +++ b/packages/scoring/src/audit.ts @@ -9,6 +9,7 @@ */ import type { AuditReport, Finding, FindingSeverity, Score, ScoringContext } from '@advance-labs/types'; import { runRules } from './engine.js'; +import { withoutNoindexPages } from './context-utils.js'; import { technicalSeoRules } from './technical-seo-rules.js'; import { aeoRules } from './aeo-rules.js'; import { generateTemplates } from './templates.js'; @@ -66,12 +67,14 @@ export async function buildAuditReport( const allFindings = categories.flatMap((c) => c.findings); const topFixes = prioritizeFixes(allFindings); const templates = generateTemplates(ctx); + const { excluded } = withoutNoindexPages(ctx); return { url: ctx.crawl.rootUrl, generatedAt: new Date().toISOString(), score, pagesCrawled: ctx.crawl.pageCount, + ...(excluded.length > 0 ? { excludedPages: excluded } : {}), topFixes, templates, meta: { diff --git a/packages/scoring/src/context-utils.ts b/packages/scoring/src/context-utils.ts index 29341e3..f607f25 100644 --- a/packages/scoring/src/context-utils.ts +++ b/packages/scoring/src/context-utils.ts @@ -148,10 +148,70 @@ export function pagesFailing(ctx: ScoringContext, pred: (page: ParsedHtml) => bo return out; } +/** Directive tokens of a robots meta / X-Robots-Tag value, lower-cased. `none` means noindex+nofollow. */ +function robotsTokens(value: string | undefined, agentOk: (agent: string) => boolean): string[] { + const out: string[] = []; + for (const raw of (value ?? '').toLowerCase().split(',')) { + // X-Robots-Tag allows a leading user-agent: "googlebot: noindex". Only count agents that + // are search engines; "otherbot: noindex" says nothing about us. + const m = /^\s*([a-z0-9._-]+)\s*:\s*(.+)$/.exec(raw); + if (m && m[1] !== undefined && m[2] !== undefined) { + if (agentOk(m[1])) out.push(m[2].trim()); + } else if (raw.trim()) out.push(raw.trim()); + } + return out.flatMap((t) => (t === 'none' ? ['noindex', 'nofollow'] : [t])); +} +const isSearchAgent = (a: string): boolean => a === 'googlebot' || a === 'bingbot'; + /** Whether a robots `` directive on a page marks it noindex. */ export function isNoindex(page: ParsedHtml): boolean { - const robots = page.meta.robots?.toLowerCase() ?? ''; - return robots.includes('noindex'); + return robotsTokens(page.meta.robots, () => true).includes('noindex'); +} + +/** Whether an `X-Robots-Tag` response header marks the page noindex. */ +export function isNoindexHeader(headers: Record | undefined): boolean { + return robotsTokens(headers?.['x-robots-tag'], isSearchAgent).includes('noindex'); +} + +/** Normalized URLs of crawled pages that are noindex by meta robots or X-Robots-Tag. */ +export function noindexUrls(ctx: ScoringContext): Set { + const out = new Set(); + for (const p of ctx.pages) if (isNoindex(p)) out.add(normalizeUrl(p.url)); + for (const c of ctx.crawl.pages) { + if (isNoindexHeader(c.headers)) { + out.add(normalizeUrl(c.url)); + out.add(normalizeUrl(c.finalUrl)); + } + } + return out; +} + +/** + * The context rules are scored against: parsed pages that opted out of indexing are removed + * (and, when `structuredData` lines up one-to-one with `pages`, their structured data too), and + * reported in `excluded`. `crawl` is left whole: robots/sitemap/redirect/broken-page rules are + * about the site and the crawl, not about indexed pages. If EVERY page is noindex the full + * context is returned untouched, since there is nothing left to score. + */ +export function withoutNoindexPages(ctx: ScoringContext): { + ctx: ScoringContext; + excluded: Array<{ url: Url; reason: 'noindex' }>; +} { + const noindex = noindexUrls(ctx); + const keep = ctx.pages.map((p) => !noindex.has(normalizeUrl(p.url))); + const excluded = ctx.pages + .filter((_, i) => !keep[i]) + .map((p) => ({ url: p.url, reason: 'noindex' as const })); + if (excluded.length === 0 || excluded.length === ctx.pages.length) return { ctx, excluded: [] }; + const aligned = ctx.structuredData.length === ctx.pages.length; + return { + ctx: { + ...ctx, + pages: ctx.pages.filter((_, i) => keep[i]), + structuredData: aligned ? ctx.structuredData.filter((_, i) => keep[i]) : ctx.structuredData, + }, + excluded, + }; } /** diff --git a/packages/scoring/src/engine.ts b/packages/scoring/src/engine.ts index e8a0187..89c4c71 100644 --- a/packages/scoring/src/engine.ts +++ b/packages/scoring/src/engine.ts @@ -22,6 +22,7 @@ import type { ScoringContext, } from '@advance-labs/types'; import { clampScore, scoreToGrade } from './grade.js'; +import { withoutNoindexPages } from './context-utils.js'; /** Human-readable labels for each category key, used in `ScoreCategory.label`. */ const CATEGORY_LABELS: Record = { @@ -118,14 +119,16 @@ export async function runRules( ): Promise<{ categories: ScoreCategory[]; score: Score }> { const byCategory = new Map(); const order: ScoreCategoryKey[] = []; + const { ctx: scored } = withoutNoindexPages(ctx); for (const rule of rules) { + const view = rule.includesNoindexPages ? ctx : scored; // ADV-175: a rule that does not apply to this page contributes nothing at all — no // finding and no weight — rather than a pass that overstates coverage or a fail that // manufactures work. - if (rule.appliesTo && !rule.appliesTo(ctx)) continue; + if (rule.appliesTo && !rule.appliesTo(view)) continue; - const outcome = await evaluateRule(rule, ctx); + const outcome = await evaluateRule(rule, view); const finding = toFinding(rule, outcome); let bucket = byCategory.get(rule.category); if (!bucket) { diff --git a/packages/scoring/src/index.ts b/packages/scoring/src/index.ts index 44d711f..bd2419d 100644 --- a/packages/scoring/src/index.ts +++ b/packages/scoring/src/index.ts @@ -23,4 +23,4 @@ export type { BuildAuditReportOptions } from './audit.js'; export { generateTemplates } from './templates.js'; // Context helpers — useful for apps building their own ad-hoc rules. -export { KEY_AI_BOTS } from './context-utils.js'; +export { KEY_AI_BOTS, withoutNoindexPages } from './context-utils.js'; diff --git a/packages/scoring/src/noindex.test.ts b/packages/scoring/src/noindex.test.ts new file mode 100644 index 0000000..b801d0d --- /dev/null +++ b/packages/scoring/src/noindex.test.ts @@ -0,0 +1,138 @@ +import { describe, expect, it } from 'vitest'; +import type { Finding, ParsedHtml, ScoringContext } from '@advance-labs/types'; +import { runRules } from './engine.js'; +import { buildAuditReport, auditRules } from './audit.js'; +import { goodContext } from './fixtures.js'; +import { isNoindex, isNoindexHeader } from './context-utils.js'; + +const LOGIN = 'https://good.example.com/login'; + +async function findings(ctx: ScoringContext): Promise> { + const { categories } = await runRules(ctx, auditRules); + return new Map(categories.flatMap((c) => c.findings).map((f) => [f.id, f])); +} + +/** goodContext plus a /login page that opts out of indexing and would fail most per-page rules. */ +function withLogin(opts: { meta?: string; header?: string; inSitemap?: boolean; sitemap?: boolean } = {}): ScoringContext { + const base = goodContext(); + const proto = base.pages[1] as ParsedHtml; + const login: ParsedHtml = { + ...proto, + url: LOGIN, + meta: { + ...proto.meta, + title: 'Sign in', + titleLength: 7, + description: undefined, + descriptionLength: 0, + canonical: undefined, + robots: opts.meta ?? (opts.header ? undefined : 'noindex, nofollow'), + }, + content: { ...proto.content, wordCount: 12 }, + }; + return { + ...base, + crawl: { + ...base.crawl, + pages: [ + ...base.crawl.pages, + { + url: LOGIN, + finalUrl: LOGIN, + status: 200, + ok: true, + headers: opts.header ? { 'x-robots-tag': opts.header } : {}, + timingMs: 100, + redirectChain: [], + depth: 1, + }, + ], + pageCount: base.crawl.pageCount + 1, + sitemap: opts.sitemap === false ? [] : opts.inSitemap ? [...base.crawl.sitemap, { loc: LOGIN }] : base.crawl.sitemap, + }, + pages: [...base.pages, login], + }; +} + +describe('noindex pages are excluded from per-page scoring', () => { + it('does not score a noindex page on title, description, thin-content or heading rules', async () => { + const baseline = await findings(goodContext()); + const f = await findings(withLogin()); + for (const id of ['tech.meta-title-length', 'tech.meta-description-present', 'tech.content-not-thin', 'tech.unique-titles']) { + expect(f.get(id)?.passed, id).toBe(baseline.get(id)?.passed); + expect(f.get(id)?.affectedUrls ?? []).not.toContain(LOGIN); + } + }); + + it('applies to X-Robots-Tag noindex too', async () => { + const f = await findings(withLogin({ header: 'noindex' })); + expect(f.get('tech.meta-title-length')?.passed).toBe(true); + expect(f.get('tech.meta-description-present')?.passed).toBe(true); + }); + + it('still records the page as crawled and lists it as excluded', async () => { + const ctx = withLogin(); + const report = await buildAuditReport(ctx, { durationMs: 1, version: 't' }); + expect(report.pagesCrawled).toBe(ctx.crawl.pageCount); + expect(report.excludedPages).toEqual([{ url: LOGIN, reason: 'noindex' }]); + expect((await buildAuditReport(goodContext(), { durationMs: 1, version: 't' })).excludedPages).toBeUndefined(); + }); + + it('scores everything when every page is noindex (nothing left to exclude into)', async () => { + const ctx = goodContext(); + ctx.pages = ctx.pages.map((p) => ({ ...p, meta: { ...p.meta, robots: 'noindex' } })); + const report = await buildAuditReport(ctx, { durationMs: 1, version: 't' }); + expect(report.excludedPages).toBeUndefined(); + }); +}); + +describe('indexability rules still see noindex pages', () => { + it('tech.indexable ignores a noindex page the sitemap leaves out (deliberate opt-out)', async () => { + expect((await findings(withLogin())).get('tech.indexable')?.passed).toBe(true); + }); + + it('tech.indexable flags a noindex page the sitemap lists (contradiction)', async () => { + const f = (await findings(withLogin({ inSitemap: true }))).get('tech.indexable'); + expect(f?.passed).toBe(false); + expect(f?.affectedUrls).toEqual([LOGIN]); + }); + + it('tech.indexable flags noindex when there is no sitemap to show intent', async () => { + const f = (await findings(withLogin({ sitemap: false }))).get('tech.indexable'); + expect(f?.passed).toBe(false); + expect(f?.affectedUrls).toEqual([LOGIN]); + }); + + it('tech.indexable always flags a noindex homepage', async () => { + const ctx = goodContext(); + ctx.pages = ctx.pages.map((p, i) => (i === 0 ? { ...p, meta: { ...p.meta, robots: 'noindex' } } : p)); + ctx.crawl.sitemap = ctx.crawl.sitemap.filter((e) => e.loc !== ctx.crawl.rootUrl); + expect((await findings(ctx)).get('tech.indexable')?.passed).toBe(false); + }); + + it('tech.indexable flags an X-Robots-Tag noindex page the sitemap lists', async () => { + const f = (await findings(withLogin({ header: 'noindex', inSitemap: true }))).get('tech.indexable'); + expect(f?.passed).toBe(false); + }); + + it('canonical-present keeps inspecting noindex pages', async () => { + const f = (await findings(withLogin())).get('tech.canonical-present'); + expect(f?.affectedUrls).toContain(LOGIN); + }); +}); + +describe('robots directive parsing', () => { + const page = (robots?: string): ParsedHtml => ({ meta: { robots } }) as unknown as ParsedHtml; + it('reads noindex, none and ignores nofollow-only', () => { + expect(isNoindex(page('noindex, nofollow'))).toBe(true); + expect(isNoindex(page('none'))).toBe(true); + expect(isNoindex(page('index, nofollow'))).toBe(false); + expect(isNoindex(page(undefined))).toBe(false); + }); + it('reads X-Robots-Tag, honouring search-engine agent prefixes only', () => { + expect(isNoindexHeader({ 'x-robots-tag': 'noindex' })).toBe(true); + expect(isNoindexHeader({ 'x-robots-tag': 'googlebot: noindex' })).toBe(true); + expect(isNoindexHeader({ 'x-robots-tag': 'otherbot: noindex' })).toBe(false); + expect(isNoindexHeader({})).toBe(false); + }); +}); diff --git a/packages/scoring/src/technical-seo-rules.ts b/packages/scoring/src/technical-seo-rules.ts index e490757..548af2a 100644 --- a/packages/scoring/src/technical-seo-rules.ts +++ b/packages/scoring/src/technical-seo-rules.ts @@ -14,6 +14,7 @@ import { firstPage, firstStructured, isNoindex, + isNoindexHeader, longRedirectChains, meanOverPages, normalizeUrl, @@ -264,8 +265,28 @@ export const technicalSeoRules: Rule[] = [ title: 'Pages are indexable (no accidental noindex)', description: 'A noindex directive removes a page from search and answer engines.', recommendation: 'Remove noindex from pages you want discovered.', + // Sees the noindex pages on purpose: finding an accidental one is this rule's whole job. + includesNoindexPages: true, evaluate: (ctx) => { - const noindexed = pagesFailing(ctx, (p) => !isNoindex(p)); + // A noindex page is a defect only if the site otherwise presents it as indexable. When a + // sitemap exists, a noindex page the sitemap does NOT list (a login or checkout page that + // was only found by following a link) is a deliberate opt-out. Flagged: the root page, + // any noindex page the sitemap lists (contradiction), and every noindex page when there + // is no sitemap to tell intent from accident. + const listed = new Set(ctx.crawl.sitemap.map((e) => normalizeUrl(e.loc))); + const headerNoindex = new Set( + ctx.crawl.pages.filter((c) => isNoindexHeader(c.headers)).flatMap((c) => [normalizeUrl(c.url), normalizeUrl(c.finalUrl)]), + ); + const root = normalizeUrl(ctx.crawl.rootUrl); + const noindexed = pagesFailing( + ctx, + (p) => { + const key = normalizeUrl(p.url); + if (!isNoindex(p) && !headerNoindex.has(key)) return true; + const deliberate = listed.size > 0 && key !== root && !listed.has(key); + return deliberate; + }, + ); return noindexed.length === 0 ? { passed: true } : { @@ -283,6 +304,9 @@ export const technicalSeoRules: Rule[] = [ title: 'Canonical URL is declared', description: 'A canonical tag consolidates duplicate URLs to one preferred address.', recommendation: 'Add to every indexable page.', + // Canonical checks keep seeing noindex pages: a canonical on a page you meant to index is + // exactly what they verify, and pages are only excluded from scoring, not from inspection. + includesNoindexPages: true, evaluate: (ctx) => { const missing = pagesFailing(ctx, (p) => Boolean(p.meta.canonical)); return missing.length === 0 ? { passed: true } : { passed: false, affectedUrls: missing }; @@ -307,6 +331,7 @@ export const technicalSeoRules: Rule[] = [ recommendation: 'Point every canonical at an absolute URL that returns 200. Check it after any move or domain change.', docsUrl: 'https://developers.google.com/search/docs/crawling-indexing/consolidate-duplicate-urls', + includesNoindexPages: true, evaluate: (ctx) => { const brokenByUrl = new Map(); for (const p of brokenPages(ctx)) brokenByUrl.set(normalizeUrl(p.url), p.status); diff --git a/packages/types/src/audit.ts b/packages/types/src/audit.ts index 2135159..1acb3f3 100644 --- a/packages/types/src/audit.ts +++ b/packages/types/src/audit.ts @@ -22,6 +22,8 @@ export interface AuditReport { generatedAt: string; score: Score; pagesCrawled: number; + /** Crawled pages left out of per-page rule scoring, with why. Present only when non-empty. */ + excludedPages?: Array<{ url: Url; reason: 'noindex' }>; /** Prioritized fix list (failed findings, highest-impact first). */ topFixes: Finding[]; /** Generated templates for missing files. */ diff --git a/packages/types/src/scoring.ts b/packages/types/src/scoring.ts index e687619..4e81aeb 100644 --- a/packages/types/src/scoring.ts +++ b/packages/types/src/scoring.ts @@ -92,5 +92,12 @@ export interface Rule { * claim we checked something we did not. */ appliesTo?: (ctx: TContext) => boolean; + /** + * Rules are scored against the pages that ask to be indexed: pages marked `noindex` (meta robots + * or X-Robots-Tag) are left out of `ctx.pages` / `ctx.structuredData`, because a page that opted + * out of search has no title, description or schema obligations. Set this on a rule whose whole + * job is to inspect indexability (it must see the noindex pages), and it gets the full context. + */ + includesNoindexPages?: boolean; evaluate: (ctx: TContext) => RuleOutcome | Promise; }