feat(agent-readiness): add public site checks

.claude-plugin/plugin.json

@@ -2,7 +2,7 @@
   "name": "seo",
   "displayName": "seo",
   "description": "Local-first SEO and AI-search diagnostics. Bundles the seo MCP server plus one SEO skill that gives an agent 50+ audit and report tools without filling its context window.",
-  "version": "0.2.38",
+  "version": "0.2.39",
   "author": {
     "name": "Ian Nuttall"
   },

apps/web/src/content/reports/crawl.ts

@@ -46,19 +46,19 @@ export const crawlReports = [
     name: 'AI agent readiness',
     category: 'crawl',
     summary:
-      'Check whether a content site gives agents stable Markdown alternatives, useful discovery files, clear identity, and predictable access.',
+      'Check whether a content site gives agents useful raw HTML, safe missing pages, stable Markdown, clear guidance, and predictable access.',
     question:
-      'Can an agent find, fetch, and read the public content without losing the document structure or guessing which routes exist?',
+      'Can an agent find, fetch, and read the public content, recover from a missing path, and find the next useful entry point?',
     useWhen: [
       'A documentation, publishing, or content site serves machine-readable versions of its pages.',
-      'You want one focused check for Markdown delivery, agent discovery, access, and identity.',
+      'You want one focused check for raw HTML, missing-page recovery, trust pages, Markdown delivery, agent discovery, access, and identity.',
     ],
     avoidWhen: [
       'You need an AI mention, citation, referral, ranking, or selection measurement.',
       'The main job is testing a public API, interactive agent application, or commerce flow.',
     ],
     evidence: [
-      'Public HTML pages, Markdown alternatives, HTTP content negotiation, route manifests, Agent Skills, llms.txt, crawler policies, and structured identity.',
+      'Raw public HTML, missing-page responses, trust pages, developer links, Markdown alternatives, Agent Skills, llms.txt, crawler policies, and structured identity.',
     ],
     methodology: [
       'Selects the content profile, checks each public representation and discovery surface, and marks unrelated capability profiles as not applicable rather than failed.',
@@ -74,6 +74,7 @@ export const crawlReports = [
       'This report has no aggregate score. A clean result cannot prove that an AI service fetched, used, mentioned, cited, ranked, or selected a page.',
     ],
     nextSteps: [
+      'Fix false success responses and empty raw documents before optional discovery improvements.',
       'Fix shared generation or delivery problems before patching individual Markdown files.',
       'Repeat the same public route scope after deployment and compare the affected checks.',
     ],

apps/web/src/content/reports/page-copy.ts

@@ -14,8 +14,8 @@ export const reportPageCopy: Record<string, ReportPageCopy> = {
   'agent-readiness': {
     title: 'AI agent readiness',
     description:
-      'Check Markdown alternatives, agent discovery, crawler access and identity across a content site. Find broken routes without inventing a readiness score.',
-    lead: 'Check whether agents can find and read a clean machine-readable version of every public page. The report tests the content-site contract directly, then keeps unrelated API, application and commerce checks out of the failure count.',
+      'Check raw HTML, missing-page recovery, trust pages, Markdown, agent discovery, crawler access and identity without inventing a readiness score.',
+    lead: 'Check whether agents can read the initial HTML, recover from a missing path, and find a clean machine-readable version of every public page. The report keeps unrelated API, application and commerce checks out of the failure count.',
   },
   'ai-readiness': {
     title: 'AI search readiness audit',

apps/web/src/content/reports/section-copy.ts

@@ -2,7 +2,7 @@ export const reportNextStepIntros: Record<string, string> = {
   'affected-urls':
     'The URL list is ready for inspection, not bulk editing. Open representative pages, confirm that the rule conflicts with their purpose, then recrawl the same scope after the fix.',
   'agent-readiness':
-    'Fix shared representation failures before patching one generated file. Repeat the same route scope after deployment and confirm the explicit Markdown URL, negotiated response, discovery files, and identity evidence now agree.',
+    'Fix false success responses and empty raw documents first. Then fix shared representation failures before patching one generated file, and repeat the same route scope after deployment.',
   'ai-readiness':
     "Start with hard access, indexability or snippet conflicts that work against the publisher's intent. Keep referral traffic and assistant mentions as separate measurements because this audit cannot observe either one.",
   'ai-referrals':

package.json

@@ -1,6 +1,6 @@
 {
   "name": "seo",
-  "version": "0.2.38",
+  "version": "0.2.39",
   "description": "The SEO command for AI agents. Audit sites and research search opportunities with local, evidence-backed reports.",
   "type": "module",
   "license": "Apache-2.0",

packages/cli/src/commands/agent-readiness.ts

@@ -68,7 +68,7 @@ export const agentReadinessCommand = defineCommand({
   meta: {
     name: 'agent-readiness',
     description:
-      'Check content-site access, Markdown alternatives, discovery, and identity for AI agents',
+      'Check raw HTML, missing pages, guidance, Markdown, discovery, and identity for AI agents',
   },
   args: {
     agentReadinessUrl: {

packages/core/src/analyze/crawler/agent-discovery-schema.ts

@@ -17,6 +17,22 @@ const agentRepresentationResponseSchema = z.object({
   error: z.string().optional(),
 })
 
+const agentPublicPageObservationSchema = z.object({
+  requestedUrl: z.string().url(),
+  finalUrl: z.string().url().optional(),
+  status: z.number().int().optional(),
+  contentType: z.string().optional(),
+  characters: z.number().int().nonnegative().optional(),
+  wordCount: z.number().int().nonnegative().optional(),
+  h1Count: z.number().int().nonnegative().optional(),
+  headingLevels: z.array(z.number().int().min(1).max(6)).optional(),
+  maximumHeadingSkip: z.number().int().nonnegative().optional(),
+  links: z.array(z.string().url()).optional(),
+  bodyLimitExceeded: z.boolean().optional(),
+  bodyLimitBytes: z.number().int().positive().optional(),
+  error: z.string().optional(),
+})
+
 export const agentDiscoverySchema = z.object({
   profile: z.literal('content'),
   profileApplicability: z.record(
@@ -119,6 +135,10 @@ export const agentDiscoverySchema = z.object({
         observedDigest: z.string().optional(),
         digestMatches: z.boolean().nullable(),
         frontmatterValid: z.boolean().nullable(),
+        whenToUseGuidance: z.boolean().nullable().optional(),
+        guidanceSource: z
+          .enum(['frontmatter-description', 'body-heading'])
+          .optional(),
         sameOrigin: z.boolean(),
         cors: z.string().optional(),
         error: z.string().optional(),
@@ -167,6 +187,8 @@ export const agentDiscoverySchema = z.object({
     nonIndexableLinks: z.array(z.string().url()),
     missingCrawlRoutes: z.array(z.string().url()),
     oversized: z.boolean(),
+    whenToUseGuidance: z.boolean().nullable().optional(),
+    guidanceSource: z.enum(['heading', 'summary']).optional(),
     discovery: z
       .object({
         source: z.enum(['html-link', 'http-link', 'path-probe']),
@@ -218,6 +240,25 @@ export const agentDiscoverySchema = z.object({
       ),
     })
     .optional(),
+  siteSurfaces: z
+    .object({
+      rawStartPage: agentPublicPageObservationSchema,
+      notFound: z.object({
+        path: z.string(),
+        html: agentPublicPageObservationSchema,
+        markdown: agentPublicPageObservationSchema,
+      }),
+      trustAnchors: z.array(
+        agentPublicPageObservationSchema.extend({
+          id: z.enum(['about', 'contact', 'privacy']),
+        }),
+      ),
+      limits: z.object({
+        bodyBytesPerResponse: z.number().int().positive(),
+        trustAnchorPages: z.number().int().nonnegative(),
+      }),
+    })
+    .optional(),
   protocolVariants: z.object({
     http: z.object({
       url: z.string().url(),

packages/core/src/analyze/crawler/agent-discovery-types.ts

@@ -73,11 +73,45 @@ export type AgentSkillObservation = {
   observedDigest?: string
   digestMatches: boolean | null
   frontmatterValid: boolean | null
+  whenToUseGuidance?: boolean | null
+  guidanceSource?: 'frontmatter-description' | 'body-heading'
   sameOrigin: boolean
   cors?: string
   error?: string
 }
 
+export type AgentPublicPageObservation = {
+  requestedUrl: string
+  finalUrl?: string
+  status?: number
+  contentType?: string
+  characters?: number
+  wordCount?: number
+  h1Count?: number
+  headingLevels?: number[]
+  maximumHeadingSkip?: number
+  links?: string[]
+  bodyLimitExceeded?: boolean
+  bodyLimitBytes?: number
+  error?: string
+}
+
+export type AgentSiteSurfaceObservation = {
+  rawStartPage: AgentPublicPageObservation
+  notFound: {
+    path: string
+    html: AgentPublicPageObservation
+    markdown: AgentPublicPageObservation
+  }
+  trustAnchors: Array<
+    AgentPublicPageObservation & { id: 'about' | 'contact' | 'privacy' }
+  >
+  limits: {
+    bodyBytesPerResponse: number
+    trustAnchorPages: number
+  }
+}
+
 export type AgentEndpointObservation = {
   id: string
   url: string
@@ -191,6 +225,8 @@ export type CrawlAgentDiscovery = {
     nonIndexableLinks: string[]
     missingCrawlRoutes: string[]
     oversized: boolean
+    whenToUseGuidance?: boolean | null
+    guidanceSource?: 'heading' | 'summary'
     discovery?: {
       source: 'html-link' | 'http-link' | 'path-probe'
       advertisedUrls: string[]
@@ -210,6 +246,7 @@ export type CrawlAgentDiscovery = {
     consistent: boolean | null
   }
   endpointDiscovery?: AgentEndpointDiscovery
+  siteSurfaces?: AgentSiteSurfaceObservation
   protocolVariants: {
     http: {
       url: string

packages/core/src/analyze/crawler/agent-discovery.ts

@@ -20,6 +20,7 @@ import type {
   MarkdownQualityObservation,
 } from './agent-discovery-types.js'
 import { inspectAgentEndpoints } from './agent-endpoints.js'
+import { inspectAgentSiteSurfaces } from './agent-site-surfaces.js'
 import { inspectLlmsTxt } from './llms-txt-discovery.js'
 
 export { fetchText, linkEntries, safeError } from './agent-discovery-http.js'
@@ -65,6 +66,26 @@ function bodyWordCount(value: string): number {
     .filter(Boolean).length
 }
 
+function whenToUseGuidance(value: string): {
+  present: boolean
+  source?: 'frontmatter-description' | 'body-heading'
+} {
+  const frontmatter = value.match(/^---\s*\n([\s\S]*?)^---\s*$/mu)?.[1] ?? ''
+  const description =
+    frontmatter.match(/^description:\s*["']?(.+?)["']?\s*$/imu)?.[1] ?? ''
+  if (
+    /\b(?:use|reach for|best (?:used|suited))\b[^.\n]{0,80}\b(?:when|for)\b/iu.test(
+      description,
+    )
+  ) {
+    return { present: true, source: 'frontmatter-description' }
+  }
+  if (/^#{1,3}\s+when to use(?:\s|$)/imu.test(value)) {
+    return { present: true, source: 'body-heading' }
+  }
+  return { present: false }
+}
+
 function repeatedProseLines(markdown: string): number {
   const counts = new Map<string, number>()
   let inFrontmatter = false
@@ -403,6 +424,7 @@ async function inspectAgentSkills(input: {
             /^---\s*\n[\s\S]*?^name:\s*\S.+$[\s\S]*?^description:\s*\S.+$[\s\S]*?^---\s*$/mu.test(
               skill.body,
             )
+          const guidance = whenToUseGuidance(skill.body)
           return {
             name,
             url,
@@ -418,6 +440,8 @@ async function inspectAgentSkills(input: {
               ? declaredDigest === observedDigest
               : null,
             frontmatterValid,
+            whenToUseGuidance: guidance.present,
+            ...(guidance.source ? { guidanceSource: guidance.source } : {}),
             sameOrigin,
           }
         } catch (error) {
@@ -560,12 +584,13 @@ export async function collectAgentDiscovery(input: {
       ),
     ),
   )
-  const [agentSkills, llmsTxt, routeManifest, endpointDiscovery] =
+  const [agentSkills, llmsTxt, routeManifest, endpointDiscovery, siteSurfaces] =
     await Promise.all([
       inspectAgentSkills({ ...input, origin }),
       inspectLlmsTxt({ ...input, origin, pages }),
       inspectRouteManifest({ ...input, origin }),
       inspectAgentEndpoints({ ...input, origin }),
+      inspectAgentSiteSurfaces({ ...input, origin }),
     ])
   const qZero = await fetchRepresentation({
     url: input.startUrl,
@@ -682,6 +707,7 @@ export async function collectAgentDiscovery(input: {
     agentSkills,
     llmsTxt,
     endpointDiscovery,
+    siteSurfaces,
     contentSignals: {
       htmlValues: [...new Set(htmlContentSignals)].sort(),
       markdownValues: [...new Set(markdownContentSignals)].sort(),