fix(llms): separate spec rules from safety limits

packages/cli/src/commands/llms.ts

@@ -16,6 +16,8 @@ import { printJson, printKeyValue } from '../utils.js'
 import { printNotes, printReportSummary } from './output.js'
 import { resolveSavedCrawlReport } from './readiness.js'
 
+const JSON_LLMS_TXT_MAX_BYTES = 64 * 1024
+
 async function writeOrPrint(path: string | undefined, content: string) {
   if (!path) {
     process.stdout.write(content)
@@ -122,7 +124,12 @@ export const llmsAuditCommand = defineCommand({
     printReportSummary({
       title: 'llms.txt audit',
       target: audit.url,
-      status: audit.issues.length > 0 ? 'warning' : 'pass',
+      status:
+        audit.issues.length > 0
+          ? 'warning'
+          : audit.dataStatus === 'complete'
+            ? 'pass'
+            : 'info',
       summary: audit.headline,
       metrics: [
         { label: 'SEO impact', value: audit.googleSearchImpact },
@@ -146,6 +153,7 @@ export const llmsAuditCommand = defineCommand({
         },
       ],
     })
+    printNotes('Caveats', audit.caveats)
     printNotes(
       'Recommended pages',
       audit.recommendedPages
@@ -168,7 +176,7 @@ export const llmsGenerateCommand = defineCommand({
     },
     'max-urls': {
       type: 'string',
-      description: 'Maximum URLs to include. Defaults to 100.',
+      description: 'Maximum URLs to include. Defaults to 250.',
     },
     'token-budget': {
       type: 'string',
@@ -193,6 +201,7 @@ export const llmsGenerateCommand = defineCommand({
     const generated = generateLlmsTxt(report, {
       maxUrls: numberArg(args['max-urls']),
       tokenBudget: numberArg(args['token-budget']),
+      maxBytes: json ? JSON_LLMS_TXT_MAX_BYTES : undefined,
       exclude: csvArg(args.exclude),
       title: stringArg(args.title),
       description: stringArg(args.description),
@@ -207,6 +216,12 @@ export const llmsGenerateCommand = defineCommand({
       printKeyValue([
         ['URLs', String(generated.includedUrls)],
         ['Estimated tokens', String(generated.estimatedTokens)],
+        [
+          'Truncated',
+          generated.limits.truncated
+            ? generated.limits.reasons.join(', ')
+            : 'No',
+        ],
       ])
     }
   },

packages/core/src/analyze/crawler/agent-discovery-http.ts

@@ -1,6 +1,18 @@
 import type { publicHttpFetch } from '../../fetch/http-client.js'
 
-const MAX_BODY_BYTES = 2_000_000
+export const AGENT_DISCOVERY_MAX_BODY_BYTES = 2_000_000
+
+class ResponseBodyLimitError extends Error {
+  readonly bytesRead: number
+  readonly limitBytes: number
+
+  constructor(bytesRead: number, limitBytes: number) {
+    super(`Response exceeds ${limitBytes} bytes.`)
+    this.name = 'ResponseBodyLimitError'
+    this.bytesRead = bytesRead
+    this.limitBytes = limitBytes
+  }
+}
 
 export function headerValue(
   headers: Record<string, string> | undefined,
@@ -66,6 +78,7 @@ export function combinedSignal(
 
 export async function readBoundedText(
   response: Awaited<ReturnType<typeof publicHttpFetch>>,
+  maxBytes = AGENT_DISCOVERY_MAX_BODY_BYTES,
 ): Promise<string> {
   if (!response.body) return ''
   const reader = response.body.getReader()
@@ -76,8 +89,8 @@ export async function readBoundedText(
       const result = await reader.read()
       if (result.done) break
       size += result.value.byteLength
-      if (size > MAX_BODY_BYTES) {
-        throw new Error(`Response exceeds ${MAX_BODY_BYTES} bytes.`)
+      if (size > maxBytes) {
+        throw new ResponseBodyLimitError(size, maxBytes)
       }
       chunks.push(result.value)
     }
@@ -94,6 +107,8 @@ export async function fetchText(input: {
   signal?: AbortSignal
   redirect?: 'follow' | 'manual'
   accept?: string
+  maxBytes?: number
+  returnOnBodyLimit?: boolean
 }) {
   const controller = combinedSignal(input.timeoutMs, input.signal)
   try {
@@ -103,7 +118,27 @@ export async function fetchText(input: {
       headers: input.accept ? { accept: input.accept } : undefined,
       signal: controller.signal,
     })
-    return { response, body: await readBoundedText(response) }
+    try {
+      const body = await readBoundedText(response, input.maxBytes)
+      return {
+        response,
+        body,
+        bodyLimitExceeded: false,
+        bodyLimitBytes: input.maxBytes ?? AGENT_DISCOVERY_MAX_BODY_BYTES,
+        bytesRead: Buffer.byteLength(body),
+      }
+    } catch (error) {
+      if (input.returnOnBodyLimit && error instanceof ResponseBodyLimitError) {
+        return {
+          response,
+          body: '',
+          bodyLimitExceeded: true,
+          bodyLimitBytes: error.limitBytes,
+          bytesRead: error.bytesRead,
+        }
+      }
+      throw error
+    }
   } finally {
     controller.cleanup()
   }

packages/core/src/analyze/crawler/agent-discovery-schema.ts

@@ -132,12 +132,21 @@ export const agentDiscoverySchema = z.object({
     status: z.number().int().optional(),
     contentType: z.string().optional(),
     bytes: z.number().int().nonnegative().optional(),
+    bytesStatus: z.enum(['exact', 'lower-bound']).optional(),
     sha256: z.string().optional(),
     repeatedHashStable: z.boolean().nullable(),
+    bodyDataStatus: z.enum(['complete', 'partial', 'unavailable']).optional(),
+    bodyLimitBytes: z.number().int().positive().optional(),
+    bodyLimitExceeded: z.boolean().optional(),
     formatValid: z.boolean().nullable().optional(),
     formatErrors: z.array(z.string()).optional(),
     headingCount: z.number().int().nonnegative(),
     totalParsedLinks: z.number().int().nonnegative(),
+    linkCheckStatus: z
+      .enum(['complete', 'partial', 'unavailable', 'not-applicable'])
+      .optional(),
+    linkCheckLimit: z.number().int().positive().optional(),
+    linksChecked: z.number().int().nonnegative().optional(),
     linkLimitReached: z.boolean(),
     links: z.array(
       z.object({

packages/core/src/analyze/crawler/agent-discovery-types.ts

@@ -169,12 +169,19 @@ export type CrawlAgentDiscovery = {
     status?: number
     contentType?: string
     bytes?: number
+    bytesStatus?: 'exact' | 'lower-bound'
     sha256?: string
     repeatedHashStable: boolean | null
+    bodyDataStatus?: AgentDiscoveryDataStatus
+    bodyLimitBytes?: number
+    bodyLimitExceeded?: boolean
     formatValid?: boolean | null
     formatErrors?: string[]
     headingCount: number
     totalParsedLinks: number
+    linkCheckStatus?: 'complete' | 'partial' | 'unavailable' | 'not-applicable'
+    linkCheckLimit?: number
+    linksChecked?: number
     linkLimitReached: boolean
     links: LlmsTxtLinkObservation[]
     invalidLinks: string[]

packages/core/src/analyze/crawler/agent-discovery.test.ts

@@ -7,7 +7,6 @@ import type { publicHttpFetch } from '../../fetch/http-client.js'
 import type { CrawlPageSnapshot } from '../monitoring/types.js'
 import { collectAgentDiscovery } from './agent-discovery.js'
 import { agentReadiness } from './agent-readiness.js'
-import { auditLlmsTxt } from './llms.js'
 import { createCrawlReport } from './report.js'
 
 const markdown = `---
@@ -944,87 +943,3 @@ test('content signals fall back to the robots.txt directive when headers are abs
     'info',
   )
 })
-
-test('llms.txt validation reports malformed, stale, redirected, off-site, non-indexable, and oversized evidence', async () => {
-  const llmsBody = `${'# Example\n\n## Start\n\n- [Home](https://example.com/)\n- [Missing](https://example.com/missing)\n- [Hidden](https://example.com/hidden)\n- [Old](https://example.com/old)\n- [External](https://other.example/resource)\n- [Malformed](https://[broken])\n\n'}${'x'.repeat(100_001)}`
-  const variantFetch = (async (
-    url: string,
-    input?: Parameters<typeof fakeFetch>[1],
-  ) => {
-    const requestedUrl = String(url)
-    if (requestedUrl === 'https://example.com/llms.txt') {
-      return response(llmsBody, 200, { 'content-type': 'text/plain' })
-    }
-    if (requestedUrl === 'https://example.com/hidden') {
-      return response(
-        '<meta content="noindex, follow" name="robots"><h1>Hidden</h1>',
-        200,
-        { 'content-type': 'text/html' },
-      )
-    }
-    if (requestedUrl === 'https://example.com/old') {
-      const redirected = response('<h1>New</h1>', 200, {
-        'content-type': 'text/html',
-      })
-      Object.defineProperty(redirected, 'redirected', { value: true })
-      Object.defineProperty(redirected, 'url', {
-        value: 'https://example.com/new',
-      })
-      return redirected
-    }
-    if (requestedUrl === 'https://other.example/resource') {
-      return response('<h1>External</h1>', 200, {
-        'content-type': 'text/html',
-      })
-    }
-    return fakeFetch(requestedUrl, input)
-  }) as typeof publicHttpFetch
-
-  const discovery = await collectAgentDiscovery({
-    startUrl: 'https://example.com/',
-    pages: [page],
-    timeoutMs: 1_000,
-    fetch: variantFetch,
-  })
-
-  assert.equal(discovery.llmsTxt.oversized, true)
-  assert.equal(discovery.llmsTxt.formatValid, false)
-  assert.deepEqual(discovery.llmsTxt.invalidLinks, ['https://[broken]'])
-  assert.deepEqual(discovery.llmsTxt.offSiteLinks, [
-    'https://other.example/resource',
-  ])
-  assert.deepEqual(discovery.llmsTxt.redirectedLinks, [
-    'https://example.com/old',
-  ])
-  assert.deepEqual(discovery.llmsTxt.nonIndexableLinks, [
-    'https://example.com/hidden',
-  ])
-  assert.deepEqual(discovery.llmsTxt.missingCrawlRoutes, [
-    'https://example.com/hidden',
-    'https://example.com/missing',
-    'https://example.com/old',
-  ])
-
-  const crawl = createCrawlReport({
-    config: { url: 'https://example.com/' },
-    pages: [page],
-  }) as ReturnType<typeof createCrawlReport> & {
-    agentDiscovery: typeof discovery
-  }
-  crawl.agentDiscovery = discovery
-  const audit = auditLlmsTxt(crawl)
-  assert.equal(audit.exists, true)
-  assert.equal(
-    audit.issues.some((issue) => issue.id === 'llms-v2-format'),
-    true,
-  )
-  assert.equal(
-    audit.issues.some((issue) => issue.id === 'llms-broken-links'),
-    true,
-  )
-  const readiness = agentReadiness(crawl)
-  assert.equal(
-    readiness.checks.find((item) => item.id === 'llms-txt')?.status,
-    'warning',
-  )
-})

packages/core/src/analyze/crawler/agent-readiness.ts

@@ -195,20 +195,34 @@ function discoveryChecks(
   const llmsBroken = llms.links.filter(
     (link) => !link.status || link.status < 200 || link.status >= 400,
   )
-  const llmsValid =
+  const llmsBodyStatus =
+    llms.bodyDataStatus ?? (llms.error ? 'unavailable' : 'complete')
+  const llmsLinkCheckStatus =
+    llms.linkCheckStatus ?? (llms.linkLimitReached ? 'partial' : 'complete')
+  const llmsHasProblems =
     llms.exists &&
-    /^\s*(?:text\/plain|text\/markdown)\b/iu.test(llms.contentType ?? '') &&
-    llms.headingCount > 0 &&
-    llms.links.length > 0 &&
-    !llms.linkLimitReached &&
-    !llms.oversized &&
-    llms.invalidLinks.length === 0 &&
-    llms.duplicateLinks.length === 0 &&
-    llms.redirectedLinks.length === 0 &&
-    llms.nonIndexableLinks.length === 0 &&
-    llms.missingCrawlRoutes.length === 0 &&
-    llmsBroken.length === 0 &&
-    llms.repeatedHashStable === true
+    (!/^\s*(?:text\/plain|text\/markdown)\b/iu.test(llms.contentType ?? '') ||
+      llms.formatValid === false ||
+      llms.invalidLinks.length > 0 ||
+      llms.duplicateLinks.length > 0 ||
+      llms.redirectedLinks.length > 0 ||
+      llms.nonIndexableLinks.length > 0 ||
+      llms.missingCrawlRoutes.length > 0 ||
+      llmsBroken.length > 0 ||
+      llms.repeatedHashStable === false)
+  const llmsReviewPartial =
+    llms.exists &&
+    (llmsBodyStatus !== 'complete' ||
+      llmsLinkCheckStatus === 'partial' ||
+      llmsLinkCheckStatus === 'unavailable' ||
+      llms.repeatedHashStable === null)
+  const llmsStatus = !llms.exists
+    ? 'info'
+    : llmsHasProblems
+      ? 'warning'
+      : llmsReviewPartial
+        ? 'info'
+        : 'pass'
   return [
     check('discovery', {
       id: 'agent-skills',
@@ -225,24 +239,35 @@ function discoveryChecks(
     }),
     check('discovery', {
       id: 'llms-txt',
-      status: llms.exists ? (llmsValid ? 'pass' : 'warning') : 'info',
+      status: llmsStatus,
       title: llms.exists
-        ? llmsValid
-          ? 'llms.txt is short, stable, and its links resolve'
-          : 'llms.txt exists but needs a content or link review'
+        ? llmsHasProblems
+          ? 'llms.txt needs a content or link review'
+          : llmsReviewPartial
+            ? 'llms.txt was found, but the review is partial'
+            : 'llms.txt format and checked links passed review'
         : 'llms.txt is not published',
       plainEnglish: llms.exists
-        ? `${llms.links.length} declared links were checked. ${llms.invalidLinks.length} were malformed, ${llms.duplicateLinks.length} were duplicated, ${llms.redirectedLinks.length} redirected, ${llms.nonIndexableLinks.length} reached non-indexable pages, ${llmsBroken.length} did not resolve, and ${llms.missingCrawlRoutes.length} were missing from the crawl inventory. ${llms.offSiteLinks.length} linked to other sites.`
+        ? `${llms.linksChecked ?? llms.links.length} of ${llms.totalParsedLinks} declared links were checked. ${llms.invalidLinks.length} were malformed, ${llms.duplicateLinks.length} were duplicated, ${llms.redirectedLinks.length} redirected, ${llms.nonIndexableLinks.length} reached non-indexable pages, ${llmsBroken.length} did not resolve, and ${llms.missingCrawlRoutes.length} were missing from the crawl inventory. ${llms.offSiteLinks.length} linked to other sites.`
         : 'llms.txt is optional and its absence is not a search ranking problem.',
       action: llms.exists
-        ? 'Keep the file curated, deterministic, and limited to useful entry points whose links still resolve.'
+        ? llmsReviewPartial && !llmsHasProblems
+          ? 'Review the untested body or links before you treat the file as fully checked.'
+          : 'Keep the file curated, deterministic, and limited to useful entry points whose links still resolve.'
         : 'Add it only when an intended consumer uses it. Do not treat it as a Google ranking requirement.',
       evidence: {
         status: llms.status,
         contentType: llms.contentType,
         bytes: llms.bytes,
+        bytesStatus: llms.bytesStatus,
+        bodyDataStatus: llmsBodyStatus,
+        bodyLimitBytes: llms.bodyLimitBytes,
+        bodyLimitExceeded: llms.bodyLimitExceeded,
         oversized: llms.oversized,
         totalParsedLinks: llms.totalParsedLinks,
+        linkCheckStatus: llmsLinkCheckStatus,
+        linkCheckLimit: llms.linkCheckLimit,
+        linksChecked: llms.linksChecked ?? llms.links.length,
         linkLimitReached: llms.linkLimitReached,
         links: llms.links.length,
         invalidLinks: llms.invalidLinks,