diff --git a/packages/cli/src/commands/llms.ts b/packages/cli/src/commands/llms.ts index 709c4eea..981e6c6b 100644 --- a/packages/cli/src/commands/llms.ts +++ b/packages/cli/src/commands/llms.ts @@ -16,6 +16,8 @@ import { printJson, printKeyValue } from '../utils.js' import { printNotes, printReportSummary } from './output.js' import { resolveSavedCrawlReport } from './readiness.js' +const JSON_LLMS_TXT_MAX_BYTES = 64 * 1024 + async function writeOrPrint(path: string | undefined, content: string) { if (!path) { process.stdout.write(content) @@ -122,7 +124,12 @@ export const llmsAuditCommand = defineCommand({ printReportSummary({ title: 'llms.txt audit', target: audit.url, - status: audit.issues.length > 0 ? 'warning' : 'pass', + status: + audit.issues.length > 0 + ? 'warning' + : audit.dataStatus === 'complete' + ? 'pass' + : 'info', summary: audit.headline, metrics: [ { label: 'SEO impact', value: audit.googleSearchImpact }, @@ -146,6 +153,7 @@ export const llmsAuditCommand = defineCommand({ }, ], }) + printNotes('Caveats', audit.caveats) printNotes( 'Recommended pages', audit.recommendedPages @@ -168,7 +176,7 @@ export const llmsGenerateCommand = defineCommand({ }, 'max-urls': { type: 'string', - description: 'Maximum URLs to include. Defaults to 100.', + description: 'Maximum URLs to include. Defaults to 250.', }, 'token-budget': { type: 'string', @@ -193,6 +201,7 @@ export const llmsGenerateCommand = defineCommand({ const generated = generateLlmsTxt(report, { maxUrls: numberArg(args['max-urls']), tokenBudget: numberArg(args['token-budget']), + maxBytes: json ? JSON_LLMS_TXT_MAX_BYTES : undefined, exclude: csvArg(args.exclude), title: stringArg(args.title), description: stringArg(args.description), @@ -207,6 +216,12 @@ export const llmsGenerateCommand = defineCommand({ printKeyValue([ ['URLs', String(generated.includedUrls)], ['Estimated tokens', String(generated.estimatedTokens)], + [ + 'Truncated', + generated.limits.truncated + ? generated.limits.reasons.join(', ') + : 'No', + ], ]) } }, diff --git a/packages/core/src/analyze/crawler/agent-discovery-http.ts b/packages/core/src/analyze/crawler/agent-discovery-http.ts index ea3a2b3a..9b45c4ca 100644 --- a/packages/core/src/analyze/crawler/agent-discovery-http.ts +++ b/packages/core/src/analyze/crawler/agent-discovery-http.ts @@ -1,6 +1,18 @@ import type { publicHttpFetch } from '../../fetch/http-client.js' -const MAX_BODY_BYTES = 2_000_000 +export const AGENT_DISCOVERY_MAX_BODY_BYTES = 2_000_000 + +class ResponseBodyLimitError extends Error { + readonly bytesRead: number + readonly limitBytes: number + + constructor(bytesRead: number, limitBytes: number) { + super(`Response exceeds ${limitBytes} bytes.`) + this.name = 'ResponseBodyLimitError' + this.bytesRead = bytesRead + this.limitBytes = limitBytes + } +} export function headerValue( headers: Record | undefined, @@ -66,6 +78,7 @@ export function combinedSignal( export async function readBoundedText( response: Awaited>, + maxBytes = AGENT_DISCOVERY_MAX_BODY_BYTES, ): Promise { if (!response.body) return '' const reader = response.body.getReader() @@ -76,8 +89,8 @@ export async function readBoundedText( const result = await reader.read() if (result.done) break size += result.value.byteLength - if (size > MAX_BODY_BYTES) { - throw new Error(`Response exceeds ${MAX_BODY_BYTES} bytes.`) + if (size > maxBytes) { + throw new ResponseBodyLimitError(size, maxBytes) } chunks.push(result.value) } @@ -94,6 +107,8 @@ export async function fetchText(input: { signal?: AbortSignal redirect?: 'follow' | 'manual' accept?: string + maxBytes?: number + returnOnBodyLimit?: boolean }) { const controller = combinedSignal(input.timeoutMs, input.signal) try { @@ -103,7 +118,27 @@ export async function fetchText(input: { headers: input.accept ? { accept: input.accept } : undefined, signal: controller.signal, }) - return { response, body: await readBoundedText(response) } + try { + const body = await readBoundedText(response, input.maxBytes) + return { + response, + body, + bodyLimitExceeded: false, + bodyLimitBytes: input.maxBytes ?? AGENT_DISCOVERY_MAX_BODY_BYTES, + bytesRead: Buffer.byteLength(body), + } + } catch (error) { + if (input.returnOnBodyLimit && error instanceof ResponseBodyLimitError) { + return { + response, + body: '', + bodyLimitExceeded: true, + bodyLimitBytes: error.limitBytes, + bytesRead: error.bytesRead, + } + } + throw error + } } finally { controller.cleanup() } diff --git a/packages/core/src/analyze/crawler/agent-discovery-schema.ts b/packages/core/src/analyze/crawler/agent-discovery-schema.ts index d9b5b114..0e061627 100644 --- a/packages/core/src/analyze/crawler/agent-discovery-schema.ts +++ b/packages/core/src/analyze/crawler/agent-discovery-schema.ts @@ -132,12 +132,21 @@ export const agentDiscoverySchema = z.object({ status: z.number().int().optional(), contentType: z.string().optional(), bytes: z.number().int().nonnegative().optional(), + bytesStatus: z.enum(['exact', 'lower-bound']).optional(), sha256: z.string().optional(), repeatedHashStable: z.boolean().nullable(), + bodyDataStatus: z.enum(['complete', 'partial', 'unavailable']).optional(), + bodyLimitBytes: z.number().int().positive().optional(), + bodyLimitExceeded: z.boolean().optional(), formatValid: z.boolean().nullable().optional(), formatErrors: z.array(z.string()).optional(), headingCount: z.number().int().nonnegative(), totalParsedLinks: z.number().int().nonnegative(), + linkCheckStatus: z + .enum(['complete', 'partial', 'unavailable', 'not-applicable']) + .optional(), + linkCheckLimit: z.number().int().positive().optional(), + linksChecked: z.number().int().nonnegative().optional(), linkLimitReached: z.boolean(), links: z.array( z.object({ diff --git a/packages/core/src/analyze/crawler/agent-discovery-types.ts b/packages/core/src/analyze/crawler/agent-discovery-types.ts index 1dcd2d53..20a7963f 100644 --- a/packages/core/src/analyze/crawler/agent-discovery-types.ts +++ b/packages/core/src/analyze/crawler/agent-discovery-types.ts @@ -169,12 +169,19 @@ export type CrawlAgentDiscovery = { status?: number contentType?: string bytes?: number + bytesStatus?: 'exact' | 'lower-bound' sha256?: string repeatedHashStable: boolean | null + bodyDataStatus?: AgentDiscoveryDataStatus + bodyLimitBytes?: number + bodyLimitExceeded?: boolean formatValid?: boolean | null formatErrors?: string[] headingCount: number totalParsedLinks: number + linkCheckStatus?: 'complete' | 'partial' | 'unavailable' | 'not-applicable' + linkCheckLimit?: number + linksChecked?: number linkLimitReached: boolean links: LlmsTxtLinkObservation[] invalidLinks: string[] diff --git a/packages/core/src/analyze/crawler/agent-discovery.test.ts b/packages/core/src/analyze/crawler/agent-discovery.test.ts index 17a35576..8ccf7391 100644 --- a/packages/core/src/analyze/crawler/agent-discovery.test.ts +++ b/packages/core/src/analyze/crawler/agent-discovery.test.ts @@ -7,7 +7,6 @@ import type { publicHttpFetch } from '../../fetch/http-client.js' import type { CrawlPageSnapshot } from '../monitoring/types.js' import { collectAgentDiscovery } from './agent-discovery.js' import { agentReadiness } from './agent-readiness.js' -import { auditLlmsTxt } from './llms.js' import { createCrawlReport } from './report.js' const markdown = `--- @@ -944,87 +943,3 @@ test('content signals fall back to the robots.txt directive when headers are abs 'info', ) }) - -test('llms.txt validation reports malformed, stale, redirected, off-site, non-indexable, and oversized evidence', async () => { - const llmsBody = `${'# Example\n\n## Start\n\n- [Home](https://example.com/)\n- [Missing](https://example.com/missing)\n- [Hidden](https://example.com/hidden)\n- [Old](https://example.com/old)\n- [External](https://other.example/resource)\n- [Malformed](https://[broken])\n\n'}${'x'.repeat(100_001)}` - const variantFetch = (async ( - url: string, - input?: Parameters[1], - ) => { - const requestedUrl = String(url) - if (requestedUrl === 'https://example.com/llms.txt') { - return response(llmsBody, 200, { 'content-type': 'text/plain' }) - } - if (requestedUrl === 'https://example.com/hidden') { - return response( - '

Hidden

', - 200, - { 'content-type': 'text/html' }, - ) - } - if (requestedUrl === 'https://example.com/old') { - const redirected = response('

New

', 200, { - 'content-type': 'text/html', - }) - Object.defineProperty(redirected, 'redirected', { value: true }) - Object.defineProperty(redirected, 'url', { - value: 'https://example.com/new', - }) - return redirected - } - if (requestedUrl === 'https://other.example/resource') { - return response('

External

', 200, { - 'content-type': 'text/html', - }) - } - return fakeFetch(requestedUrl, input) - }) as typeof publicHttpFetch - - const discovery = await collectAgentDiscovery({ - startUrl: 'https://example.com/', - pages: [page], - timeoutMs: 1_000, - fetch: variantFetch, - }) - - assert.equal(discovery.llmsTxt.oversized, true) - assert.equal(discovery.llmsTxt.formatValid, false) - assert.deepEqual(discovery.llmsTxt.invalidLinks, ['https://[broken]']) - assert.deepEqual(discovery.llmsTxt.offSiteLinks, [ - 'https://other.example/resource', - ]) - assert.deepEqual(discovery.llmsTxt.redirectedLinks, [ - 'https://example.com/old', - ]) - assert.deepEqual(discovery.llmsTxt.nonIndexableLinks, [ - 'https://example.com/hidden', - ]) - assert.deepEqual(discovery.llmsTxt.missingCrawlRoutes, [ - 'https://example.com/hidden', - 'https://example.com/missing', - 'https://example.com/old', - ]) - - const crawl = createCrawlReport({ - config: { url: 'https://example.com/' }, - pages: [page], - }) as ReturnType & { - agentDiscovery: typeof discovery - } - crawl.agentDiscovery = discovery - const audit = auditLlmsTxt(crawl) - assert.equal(audit.exists, true) - assert.equal( - audit.issues.some((issue) => issue.id === 'llms-v2-format'), - true, - ) - assert.equal( - audit.issues.some((issue) => issue.id === 'llms-broken-links'), - true, - ) - const readiness = agentReadiness(crawl) - assert.equal( - readiness.checks.find((item) => item.id === 'llms-txt')?.status, - 'warning', - ) -}) diff --git a/packages/core/src/analyze/crawler/agent-readiness.ts b/packages/core/src/analyze/crawler/agent-readiness.ts index a2a5d9d5..c96ab39f 100644 --- a/packages/core/src/analyze/crawler/agent-readiness.ts +++ b/packages/core/src/analyze/crawler/agent-readiness.ts @@ -195,20 +195,34 @@ function discoveryChecks( const llmsBroken = llms.links.filter( (link) => !link.status || link.status < 200 || link.status >= 400, ) - const llmsValid = + const llmsBodyStatus = + llms.bodyDataStatus ?? (llms.error ? 'unavailable' : 'complete') + const llmsLinkCheckStatus = + llms.linkCheckStatus ?? (llms.linkLimitReached ? 'partial' : 'complete') + const llmsHasProblems = llms.exists && - /^\s*(?:text\/plain|text\/markdown)\b/iu.test(llms.contentType ?? '') && - llms.headingCount > 0 && - llms.links.length > 0 && - !llms.linkLimitReached && - !llms.oversized && - llms.invalidLinks.length === 0 && - llms.duplicateLinks.length === 0 && - llms.redirectedLinks.length === 0 && - llms.nonIndexableLinks.length === 0 && - llms.missingCrawlRoutes.length === 0 && - llmsBroken.length === 0 && - llms.repeatedHashStable === true + (!/^\s*(?:text\/plain|text\/markdown)\b/iu.test(llms.contentType ?? '') || + llms.formatValid === false || + llms.invalidLinks.length > 0 || + llms.duplicateLinks.length > 0 || + llms.redirectedLinks.length > 0 || + llms.nonIndexableLinks.length > 0 || + llms.missingCrawlRoutes.length > 0 || + llmsBroken.length > 0 || + llms.repeatedHashStable === false) + const llmsReviewPartial = + llms.exists && + (llmsBodyStatus !== 'complete' || + llmsLinkCheckStatus === 'partial' || + llmsLinkCheckStatus === 'unavailable' || + llms.repeatedHashStable === null) + const llmsStatus = !llms.exists + ? 'info' + : llmsHasProblems + ? 'warning' + : llmsReviewPartial + ? 'info' + : 'pass' return [ check('discovery', { id: 'agent-skills', @@ -225,24 +239,35 @@ function discoveryChecks( }), check('discovery', { id: 'llms-txt', - status: llms.exists ? (llmsValid ? 'pass' : 'warning') : 'info', + status: llmsStatus, title: llms.exists - ? llmsValid - ? 'llms.txt is short, stable, and its links resolve' - : 'llms.txt exists but needs a content or link review' + ? llmsHasProblems + ? 'llms.txt needs a content or link review' + : llmsReviewPartial + ? 'llms.txt was found, but the review is partial' + : 'llms.txt format and checked links passed review' : 'llms.txt is not published', plainEnglish: llms.exists - ? `${llms.links.length} declared links were checked. ${llms.invalidLinks.length} were malformed, ${llms.duplicateLinks.length} were duplicated, ${llms.redirectedLinks.length} redirected, ${llms.nonIndexableLinks.length} reached non-indexable pages, ${llmsBroken.length} did not resolve, and ${llms.missingCrawlRoutes.length} were missing from the crawl inventory. ${llms.offSiteLinks.length} linked to other sites.` + ? `${llms.linksChecked ?? llms.links.length} of ${llms.totalParsedLinks} declared links were checked. ${llms.invalidLinks.length} were malformed, ${llms.duplicateLinks.length} were duplicated, ${llms.redirectedLinks.length} redirected, ${llms.nonIndexableLinks.length} reached non-indexable pages, ${llmsBroken.length} did not resolve, and ${llms.missingCrawlRoutes.length} were missing from the crawl inventory. ${llms.offSiteLinks.length} linked to other sites.` : 'llms.txt is optional and its absence is not a search ranking problem.', action: llms.exists - ? 'Keep the file curated, deterministic, and limited to useful entry points whose links still resolve.' + ? llmsReviewPartial && !llmsHasProblems + ? 'Review the untested body or links before you treat the file as fully checked.' + : 'Keep the file curated, deterministic, and limited to useful entry points whose links still resolve.' : 'Add it only when an intended consumer uses it. Do not treat it as a Google ranking requirement.', evidence: { status: llms.status, contentType: llms.contentType, bytes: llms.bytes, + bytesStatus: llms.bytesStatus, + bodyDataStatus: llmsBodyStatus, + bodyLimitBytes: llms.bodyLimitBytes, + bodyLimitExceeded: llms.bodyLimitExceeded, oversized: llms.oversized, totalParsedLinks: llms.totalParsedLinks, + linkCheckStatus: llmsLinkCheckStatus, + linkCheckLimit: llms.linkCheckLimit, + linksChecked: llms.linksChecked ?? llms.links.length, linkLimitReached: llms.linkLimitReached, links: llms.links.length, invalidLinks: llms.invalidLinks, diff --git a/packages/core/src/analyze/crawler/knowledge-readiness.test.ts b/packages/core/src/analyze/crawler/knowledge-readiness.test.ts index 6b5b4c16..dd216cf2 100644 --- a/packages/core/src/analyze/crawler/knowledge-readiness.test.ts +++ b/packages/core/src/analyze/crawler/knowledge-readiness.test.ts @@ -445,7 +445,24 @@ test('llms v2 validation rejects non-link section entries', () => { ) }) -test('llms generator stays within v2 link and file limits', () => { +test('llms v2 validation allows more than 12 file sections', () => { + const content = [ + '# Example', + '', + ...Array.from({ length: 13 }, (_, index) => + [ + `## Section ${index}`, + '', + `- [Page ${index}](https://example.com/page-${index}.md)`, + '', + ].join('\n'), + ), + ].join('\n') + + assert.deepEqual(validateLlmsTxtV2(content), []) +}) + +test('llms generator allows more than 100 links', () => { const report = fixtureReport() const source = report.pages[0] assert.ok(source) @@ -464,11 +481,47 @@ test('llms generator stays within v2 link and file limits', () => { tokenBudget: 100_000, }) - assert.equal(generated.includedUrls, 100) - assert.ok(Buffer.byteLength(generated.content) < 100_000) + assert.equal(generated.includedUrls, 150) + assert.equal(generated.limits.maxUrls, 250) + assert.equal(generated.limits.truncated, false) assert.deepEqual(validateLlmsTxtV2(generated.content), []) }) +test('llms generator uses an explicit output budget above 100,000 bytes', () => { + const report = fixtureReport() + const source = report.pages[0] + assert.ok(source) + report.pages = Array.from({ length: 400 }, (_, index) => ({ + ...source, + url: `https://example.com/docs/page-${index}`, + finalUrl: `https://example.com/docs/page-${index}`, + title: `Guide ${index} ${'title '.repeat(20)}`, + metaDescription: `Description ${index} ${'detail '.repeat(30)}`, + contentHash: `hash-${index}`, + })) + report.summary.crawledUrls = report.pages.length + report.summary.discoveredUrls = report.pages.length + + const complete = generateLlmsTxt(report, { + maxUrls: 400, + tokenBudget: 500_000, + }) + assert.ok(Buffer.byteLength(complete.content) > 100_000) + assert.equal(complete.includedUrls, 400) + assert.equal(complete.limits.truncated, false) + assert.deepEqual(validateLlmsTxtV2(complete.content), []) + + const bounded = generateLlmsTxt(report, { + maxUrls: 400, + tokenBudget: 500_000, + maxBytes: 4_096, + }) + assert.ok(Buffer.byteLength(bounded.content) <= 4_096) + assert.equal(bounded.limits.truncated, true) + assert.deepEqual(bounded.limits.reasons, ['maxBytes']) + assert.deepEqual(validateLlmsTxtV2(bounded.content), []) +}) + test('llms.txt remains an informational AI-search observation', () => { const missing = fixtureReport() const present = fixtureReport() diff --git a/packages/core/src/analyze/crawler/llms-discovery.test.ts b/packages/core/src/analyze/crawler/llms-discovery.test.ts new file mode 100644 index 00000000..7a80b864 --- /dev/null +++ b/packages/core/src/analyze/crawler/llms-discovery.test.ts @@ -0,0 +1,376 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import type { publicHttpFetch } from '../../fetch/http-client.js' +import type { CrawlPageSnapshot } from '../monitoring/types.js' +import { collectAgentDiscovery } from './agent-discovery.js' +import { agentReadiness } from './agent-readiness.js' +import { auditLlmsTxt } from './llms.js' +import { createCrawlReport } from './report.js' + +function response( + body: string, + status = 200, + headers: Record = {}, +): Response { + return new Response(body, { status, headers }) +} + +const fakeFetch = (async (url: string) => { + if (url === 'https://example.com/llms.txt') { + return response( + '# Example\n\n> Entry points.\n\n## Start\n\n- [Home](https://example.com/): The home page.\n', + 200, + { 'content-type': 'text/plain' }, + ) + } + if (url === 'https://example.com/') { + return response('

Example

', 200, { + 'content-type': 'text/html; charset=utf-8', + }) + } + return response('', 404, { 'content-type': 'text/plain' }) +}) as typeof publicHttpFetch + +const page: CrawlPageSnapshot = { + url: 'https://example.com/', + finalUrl: 'https://example.com/', + status: 200, + contentType: 'text/html; charset=utf-8', + title: 'Example', + h1: 'Example', + h1Count: 1, + indexable: true, + wordCount: 20, + contentHash: 'html', + contentSample: 'Example page.', + outgoingInternalCount: 0, +} + +test('llms.txt validation reports malformed, stale, redirected, off-site, and non-indexable evidence in a large file', async () => { + const llmsBody = `${'# Example\n\n## Start\n\n- [Home](https://example.com/)\n- [Missing](https://example.com/missing)\n- [Hidden](https://example.com/hidden)\n- [Old](https://example.com/old)\n- [External](https://other.example/resource)\n- [Malformed](https://[broken])\n\n'}${'x'.repeat(100_001)}` + const variantFetch = (async ( + url: string, + input?: Parameters[1], + ) => { + const requestedUrl = String(url) + if (requestedUrl === 'https://example.com/llms.txt') { + return response(llmsBody, 200, { 'content-type': 'text/plain' }) + } + if (requestedUrl === 'https://example.com/hidden') { + return response( + '

Hidden

', + 200, + { 'content-type': 'text/html' }, + ) + } + if (requestedUrl === 'https://example.com/old') { + const redirected = response('

New

', 200, { + 'content-type': 'text/html', + }) + Object.defineProperty(redirected, 'redirected', { value: true }) + Object.defineProperty(redirected, 'url', { + value: 'https://example.com/new', + }) + return redirected + } + if (requestedUrl === 'https://other.example/resource') { + return response('

External

', 200, { + 'content-type': 'text/html', + }) + } + return fakeFetch(requestedUrl, input) + }) as typeof publicHttpFetch + + const discovery = await collectAgentDiscovery({ + startUrl: 'https://example.com/', + pages: [page], + timeoutMs: 1_000, + fetch: variantFetch, + }) + + assert.ok((discovery.llmsTxt.bytes ?? 0) > 100_000) + assert.equal(discovery.llmsTxt.oversized, false) + assert.equal(discovery.llmsTxt.bodyLimitExceeded, false) + assert.equal(discovery.llmsTxt.formatValid, false) + assert.deepEqual(discovery.llmsTxt.invalidLinks, ['https://[broken]']) + assert.deepEqual(discovery.llmsTxt.offSiteLinks, [ + 'https://other.example/resource', + ]) + assert.deepEqual(discovery.llmsTxt.redirectedLinks, [ + 'https://example.com/old', + ]) + assert.deepEqual(discovery.llmsTxt.nonIndexableLinks, [ + 'https://example.com/hidden', + ]) + assert.deepEqual(discovery.llmsTxt.missingCrawlRoutes, [ + 'https://example.com/hidden', + 'https://example.com/missing', + 'https://example.com/old', + ]) + + const crawl = createCrawlReport({ + config: { url: 'https://example.com/' }, + pages: [page], + }) as ReturnType & { + agentDiscovery: typeof discovery + } + crawl.agentDiscovery = discovery + const audit = auditLlmsTxt(crawl) + assert.equal(audit.exists, true) + assert.equal( + audit.issues.some((issue) => issue.id === 'llms-v2-format'), + true, + ) + assert.equal( + audit.issues.some((issue) => issue.id === 'llms-broken-links'), + true, + ) + assert.equal( + audit.issues.some((issue) => issue.id === 'llms-file-size'), + false, + ) + const readiness = agentReadiness(crawl) + assert.equal( + readiness.checks.find((item) => item.id === 'llms-txt')?.status, + 'warning', + ) +}) + +test('llms.txt audit keeps a capped link check as partial evidence', async () => { + const links = Array.from( + { length: 150 }, + (_, index) => + `- [Page ${index}](https://example.com/docs/page-${index}.md): Guide ${index}.`, + ).join('\n') + const llmsBody = `# Example\n\n> Entry points.\n\n## Docs\n\n${links}\n` + const variantFetch = (async ( + url: string, + input?: Parameters[1], + ) => { + const requestedUrl = String(url) + if (requestedUrl === 'https://example.com/llms.txt') { + return response(llmsBody, 200, { 'content-type': 'text/plain' }) + } + if (/^https:\/\/example\.com\/docs\/page-\d+\.md$/u.test(requestedUrl)) { + return response('# Guide', 200, { 'content-type': 'text/markdown' }) + } + return fakeFetch(requestedUrl, input) + }) as typeof publicHttpFetch + + const discovery = await collectAgentDiscovery({ + startUrl: 'https://example.com/', + pages: [page], + timeoutMs: 1_000, + fetch: variantFetch, + }) + assert.equal(discovery.llmsTxt.formatValid, true) + assert.equal(discovery.llmsTxt.totalParsedLinks, 150) + assert.equal(discovery.llmsTxt.linksChecked, 100) + assert.equal(discovery.llmsTxt.linkCheckStatus, 'partial') + + const crawl = createCrawlReport({ + config: { url: 'https://example.com/' }, + pages: [ + page, + { + ...page, + url: 'https://example.com/a', + finalUrl: 'https://example.com/a', + }, + { + ...page, + url: 'https://example.com/b', + finalUrl: 'https://example.com/b', + }, + ], + }) as ReturnType & { + agentDiscovery: typeof discovery + } + crawl.agentDiscovery = discovery + + const audit = auditLlmsTxt(crawl) + assert.equal(audit.dataStatus, 'partial') + assert.equal(audit.linkCheck.status, 'partial') + assert.equal(audit.linkCheck.checkedLinks, 100) + assert.equal(audit.linkCheck.totalLinks, 150) + assert.equal(audit.issues.length, 0) + assert.match(audit.caveats.join(' '), /remaining 50 links/u) + assert.equal( + agentReadiness(crawl).checks.find((item) => item.id === 'llms-txt')?.status, + 'info', + ) +}) + +test('llms.txt files above 100,000 bytes can pass format and link checks', async () => { + const llmsBody = `# Example\n\n${'Useful context. '.repeat(8_000)}\n\n## Start\n\n- [Home](https://example.com/): Home.\n` + const variantFetch = (async ( + url: string, + input?: Parameters[1], + ) => + String(url) === 'https://example.com/llms.txt' + ? response(llmsBody, 200, { 'content-type': 'text/plain' }) + : fakeFetch(String(url), input)) as typeof publicHttpFetch + + const discovery = await collectAgentDiscovery({ + startUrl: 'https://example.com/', + pages: [page], + timeoutMs: 1_000, + fetch: variantFetch, + }) + assert.ok((discovery.llmsTxt.bytes ?? 0) > 100_000) + assert.equal(discovery.llmsTxt.bodyDataStatus, 'complete') + assert.equal(discovery.llmsTxt.bodyLimitExceeded, false) + assert.equal(discovery.llmsTxt.oversized, false) + assert.equal(discovery.llmsTxt.formatValid, true) + + const crawl = createCrawlReport({ + config: { url: 'https://example.com/' }, + pages: [ + page, + { + ...page, + url: 'https://example.com/a', + finalUrl: 'https://example.com/a', + }, + { + ...page, + url: 'https://example.com/b', + finalUrl: 'https://example.com/b', + }, + ], + }) as ReturnType & { + agentDiscovery: typeof discovery + } + crawl.agentDiscovery = discovery + const audit = auditLlmsTxt(crawl) + assert.equal(audit.dataStatus, 'complete') + assert.equal(audit.issues.length, 0) + assert.equal( + agentReadiness(crawl).checks.find((item) => item.id === 'llms-txt')?.status, + 'pass', + ) +}) + +test('llms.txt body acquisition limits preserve successful file evidence', async () => { + const llmsBody = `# Example\n\n${'x'.repeat(2_000_001)}` + const advertisedPage: CrawlPageSnapshot = { + ...page, + describedBy: ['https://example.com/llms.txt'], + } + const variantFetch = (async ( + url: string, + input?: Parameters[1], + ) => + String(url) === 'https://example.com/llms.txt' + ? response(llmsBody, 200, { 'content-type': 'text/plain' }) + : fakeFetch(String(url), input)) as typeof publicHttpFetch + + const discovery = await collectAgentDiscovery({ + startUrl: 'https://example.com/', + pages: [advertisedPage], + timeoutMs: 1_000, + fetch: variantFetch, + }) + assert.equal(discovery.llmsTxt.exists, true) + assert.equal(discovery.llmsTxt.status, 200) + assert.equal(discovery.llmsTxt.bodyDataStatus, 'unavailable') + assert.equal(discovery.llmsTxt.bodyLimitBytes, 2_000_000) + assert.equal(discovery.llmsTxt.bodyLimitExceeded, true) + assert.equal(discovery.llmsTxt.bytesStatus, 'lower-bound') + assert.ok((discovery.llmsTxt.bytes ?? 0) > 2_000_000) + assert.equal(discovery.llmsTxt.formatValid, null) + assert.equal(discovery.llmsTxt.linkCheckStatus, 'unavailable') + + const crawl = createCrawlReport({ + config: { url: 'https://example.com/' }, + pages: [ + advertisedPage, + { + ...page, + url: 'https://example.com/a', + finalUrl: 'https://example.com/a', + }, + { + ...page, + url: 'https://example.com/b', + finalUrl: 'https://example.com/b', + }, + ], + }) as ReturnType & { + agentDiscovery: typeof discovery + } + crawl.agentDiscovery = discovery + const audit = auditLlmsTxt(crawl) + assert.equal(audit.exists, true) + assert.equal(audit.dataStatus, 'partial') + assert.equal(audit.bodyEvidence.status, 'unavailable') + assert.equal(audit.bodyEvidence.limitExceeded, true) + assert.equal(audit.issues.length, 0) + assert.match(audit.caveats.join(' '), /2,000,000-byte audit limit/u) + assert.equal( + agentReadiness(crawl).checks.find((item) => item.id === 'llms-txt')?.status, + 'info', + ) +}) + +test('llms.txt keeps successful file evidence when the repeat fetch fails', async () => { + const advertisedPage: CrawlPageSnapshot = { + ...page, + describedBy: ['https://example.com/llms.txt'], + } + let llmsRequests = 0 + const variantFetch = (async ( + url: string, + input?: Parameters[1], + ) => { + if (String(url) === 'https://example.com/llms.txt') { + llmsRequests += 1 + if (llmsRequests === 2) throw new Error('Repeat request failed.') + return response( + '# Example\n\n## Start\n\n- [Home](https://example.com/): Home.\n', + 200, + { 'content-type': 'text/plain' }, + ) + } + return fakeFetch(String(url), input) + }) as typeof publicHttpFetch + + const discovery = await collectAgentDiscovery({ + startUrl: 'https://example.com/', + pages: [advertisedPage], + timeoutMs: 1_000, + fetch: variantFetch, + }) + assert.equal(discovery.llmsTxt.exists, true) + assert.equal(discovery.llmsTxt.status, 200) + assert.equal(discovery.llmsTxt.formatValid, true) + assert.equal(discovery.llmsTxt.repeatedHashStable, null) + + const crawl = createCrawlReport({ + config: { url: 'https://example.com/' }, + pages: [ + advertisedPage, + { + ...page, + url: 'https://example.com/a', + finalUrl: 'https://example.com/a', + }, + { + ...page, + url: 'https://example.com/b', + finalUrl: 'https://example.com/b', + }, + ], + }) as ReturnType & { + agentDiscovery: typeof discovery + } + crawl.agentDiscovery = discovery + const audit = auditLlmsTxt(crawl) + assert.equal(audit.exists, true) + assert.equal(audit.dataStatus, 'partial') + assert.match(audit.caveats.join(' '), /repeated fetch was unavailable/u) + assert.equal( + agentReadiness(crawl).checks.find((item) => item.id === 'llms-txt')?.status, + 'info', + ) +}) diff --git a/packages/core/src/analyze/crawler/llms-txt-discovery.ts b/packages/core/src/analyze/crawler/llms-txt-discovery.ts index b9422bbc..02ebbb27 100644 --- a/packages/core/src/analyze/crawler/llms-txt-discovery.ts +++ b/packages/core/src/analyze/crawler/llms-txt-discovery.ts @@ -3,6 +3,7 @@ import PQueue from 'p-queue' import { publicHttpFetch } from '../../fetch/http-client.js' import type { CrawlPageSnapshot } from '../monitoring/types.js' import { + AGENT_DISCOVERY_MAX_BODY_BYTES, fetchText, headerValue, linkEntries, @@ -14,8 +15,7 @@ import type { LlmsTxtLinkObservation, } from './agent-discovery-types.js' -const MAX_LLMS_LINKS = 100 -const MAX_CURATED_LLMS_BYTES = 100_000 +const LLMS_LINK_CHECK_LIMIT = 100 function sha256(value: string): string { return createHash('sha256').update(value).digest('hex') @@ -136,11 +136,6 @@ export function validateLlmsTxtV2(value: string): string[] { if (/^ {0,3}#{3,6}\s+\S/gmu.test(normalized)) { errors.push('The file can use only level-one and level-two headings.') } - const sectionCount = normalized.match(/^ {0,3}##\s+\S/gmu)?.length ?? 0 - if (sectionCount > 12) { - errors.push('The file must contain no more than 12 sections.') - } - let section: { title: string; links: number } | undefined for (const rawLine of lines) { const line = rawLine.trim() @@ -284,6 +279,7 @@ async function discoverLlmsTxt(input: { fetch: input.fetch, signal: input.signal, accept: 'text/plain,text/markdown;q=0.9', + returnOnBodyLimit: true, }) if (result.response.status >= 200 && result.response.status < 300) { return { @@ -328,29 +324,73 @@ export async function inspectLlmsTxt(input: { appliesToStartUrl: llmsAppliesToPath(url, startPath), } try { - const [first, second] = await Promise.all([ - fetchText({ + const first = await fetchText({ + url, + timeoutMs: input.timeoutMs, + fetch: input.fetch, + signal: input.signal, + accept: 'text/plain,text/markdown;q=0.9', + returnOnBodyLimit: true, + }) + const exists = first.response.status >= 200 && first.response.status < 300 + if (exists && first.bodyLimitExceeded) { + const contentLength = first.response.headers.get('content-length') + const declaredBytes = + contentLength === null ? undefined : Number(contentLength) + const hasDeclaredBytes = + declaredBytes !== undefined && + Number.isSafeInteger(declaredBytes) && + declaredBytes >= 0 + return { url, - timeoutMs: input.timeoutMs, - fetch: input.fetch, - signal: input.signal, - accept: 'text/plain,text/markdown;q=0.9', - }), - fetchText({ + exists: true, + status: first.response.status, + contentType: first.response.headers.get('content-type') ?? undefined, + bytes: hasDeclaredBytes ? declaredBytes : first.bytesRead, + bytesStatus: hasDeclaredBytes ? 'exact' : 'lower-bound', + repeatedHashStable: null, + bodyDataStatus: 'unavailable', + bodyLimitBytes: first.bodyLimitBytes, + bodyLimitExceeded: true, + formatValid: null, + formatErrors: [], + headingCount: 0, + totalParsedLinks: 0, + linkCheckStatus: 'unavailable', + linkCheckLimit: LLMS_LINK_CHECK_LIMIT, + linksChecked: 0, + linkLimitReached: false, + links: [], + invalidLinks: [], + duplicateLinks: [], + offSiteLinks: [], + redirectedLinks: [], + nonIndexableLinks: [], + missingCrawlRoutes: [], + oversized: true, + discovery: discoveryEvidence, + } + } + let second: Awaited> | undefined + try { + second = await fetchText({ url, timeoutMs: input.timeoutMs, fetch: input.fetch, signal: input.signal, accept: 'text/plain,text/markdown;q=0.9', - }), - ]) - const exists = first.response.status >= 200 && first.response.status < 300 + returnOnBodyLimit: true, + }) + } catch { + // The first response still proves that the file exists. A missing repeat + // response makes only the stability evidence unavailable. + } const parsed = markdownLinks(exists ? first.body : '') const formatErrors = exists ? validateLlmsTxtV2(first.body) : [] - const links = parsed.links.slice(0, MAX_LLMS_LINKS) + const links = parsed.links.slice(0, LLMS_LINK_CHECK_LIMIT) const linkLimitReached = parsed.links.length > links.length const counts = new Map() - for (const link of links) { + for (const link of parsed.links) { counts.set(link.url, (counts.get(link.url) ?? 0) + 1) } const duplicateLinks = [...counts] @@ -415,12 +455,26 @@ export async function inspectLlmsTxt(input: { status: first.response.status, contentType: first.response.headers.get('content-type') ?? undefined, bytes: Buffer.byteLength(first.body), + bytesStatus: 'exact', sha256: sha256(first.body), - repeatedHashStable: sha256(first.body) === sha256(second.body), + repeatedHashStable: + !second || second.bodyLimitExceeded + ? null + : sha256(first.body) === sha256(second.body), + bodyDataStatus: 'complete', + bodyLimitBytes: first.bodyLimitBytes, + bodyLimitExceeded: false, formatValid: exists ? formatErrors.length === 0 : null, formatErrors, headingCount: first.body.match(/^#{1,2}\s+\S/gmu)?.length ?? 0, totalParsedLinks: parsed.links.length, + linkCheckStatus: !exists + ? 'not-applicable' + : linkLimitReached + ? 'partial' + : 'complete', + linkCheckLimit: LLMS_LINK_CHECK_LIMIT, + linksChecked: linkObservations.length, linkLimitReached, links: linkObservations, invalidLinks: parsed.invalidLinks.sort(), @@ -440,7 +494,7 @@ export async function inspectLlmsTxt(input: { missingCrawlRoutes: [...linkedCrawlRoutes] .filter((route) => !crawlRoutes.has(route)) .sort(), - oversized: Buffer.byteLength(first.body) > MAX_CURATED_LLMS_BYTES, + oversized: false, discovery: discoveryEvidence, } } catch (error) { @@ -448,10 +502,16 @@ export async function inspectLlmsTxt(input: { url, exists: false, repeatedHashStable: null, + bodyDataStatus: 'unavailable', + bodyLimitBytes: AGENT_DISCOVERY_MAX_BODY_BYTES, + bodyLimitExceeded: false, formatValid: null, formatErrors: [], headingCount: 0, totalParsedLinks: 0, + linkCheckStatus: 'unavailable', + linkCheckLimit: LLMS_LINK_CHECK_LIMIT, + linksChecked: 0, linkLimitReached: false, links: [], invalidLinks: [], diff --git a/packages/core/src/analyze/crawler/llms.ts b/packages/core/src/analyze/crawler/llms.ts index 616f55ba..5da7782b 100644 --- a/packages/core/src/analyze/crawler/llms.ts +++ b/packages/core/src/analyze/crawler/llms.ts @@ -15,6 +15,7 @@ export type LlmsAuditIssue = { export type LlmsAuditReport = { reportId: string url: string + dataStatus: 'complete' | 'partial' | 'unavailable' exists: boolean llmsTxtUrl: string status?: number @@ -22,6 +23,20 @@ export type LlmsAuditReport = { googleSearchImpact: 'none' guidanceUrl: string headline: string + caveats: string[] + bodyEvidence: { + status: 'complete' | 'unavailable' | 'not-applicable' + bytes?: number + bytesStatus?: 'exact' | 'lower-bound' + limitBytes?: number + limitExceeded: boolean + } + linkCheck: { + status: 'complete' | 'partial' | 'unavailable' | 'not-applicable' + checkedLinks: number + totalLinks: number + limit: number + } issues: LlmsAuditIssue[] recommendedPages: Array<{ url: string @@ -34,6 +49,7 @@ export type LlmsAuditReport = { export type GenerateLlmsTxtOptions = { maxUrls?: number tokenBudget?: number + maxBytes?: number exclude?: string[] title?: string description?: string @@ -44,6 +60,14 @@ export type GeneratedLlmsTxt = { includedUrls: number estimatedTokens: number sections: Record + limits: { + candidateUrls: number + maxUrls: number + tokenBudget: number + maxBytes: number + truncated: boolean + reasons: Array<'maxUrls' | 'tokenBudget' | 'maxBytes'> + } source: { reportId: string status: CrawlReport['status'] @@ -56,8 +80,12 @@ export type GeneratedLlmsTxt = { } } -const MAX_LLMS_TXT_LINKS = 100 -const MAX_LLMS_TXT_BYTES = 100_000 +export const DEFAULT_LLMS_TXT_MAX_URLS = 250 +export const MAX_LLMS_TXT_URLS = 10_000 +export const DEFAULT_LLMS_TXT_TOKEN_BUDGET = 12_000 +export const MAX_LLMS_TXT_TOKEN_BUDGET = 500_000 +export const MIN_GENERATED_LLMS_TXT_BYTES = 4_096 +export const MAX_GENERATED_LLMS_TXT_BYTES = 2_000_000 function estimatedTokens(value: string): number { return Math.ceil(value.length / 4) @@ -148,6 +176,57 @@ function buildLlmsTxtAudit( const pages = candidatePages(report) const issues: LlmsAuditIssue[] = [] const exists = validation?.exists ?? Boolean(llmsTxt?.exists) + const bodyStatus = !exists + ? 'not-applicable' + : validation?.bodyDataStatus === 'unavailable' || + validation?.bodyLimitExceeded + ? 'unavailable' + : validation + ? 'complete' + : 'unavailable' + const linkCheckStatus = !exists + ? 'not-applicable' + : (validation?.linkCheckStatus ?? + (validation?.linkLimitReached + ? 'partial' + : validation + ? 'complete' + : 'unavailable')) + const checkedLinks = validation?.linksChecked ?? validation?.links.length ?? 0 + const totalLinks = validation?.totalParsedLinks ?? 0 + const linkCheckLimit = validation?.linkCheckLimit ?? checkedLinks + const caveats: string[] = [] + if (exists && bodyStatus === 'unavailable') { + caveats.push( + validation?.bodyLimitExceeded + ? `The file body exceeded the ${validation.bodyLimitBytes?.toLocaleString('en-GB') ?? 'configured'}-byte audit limit. Its format and links were not checked.` + : 'The file body was not available for format and link checks.', + ) + } + if (linkCheckStatus === 'partial') { + caveats.push( + `The file contains ${totalLinks.toLocaleString('en-GB')} links. This run checked the first ${checkedLinks.toLocaleString('en-GB')}. Findings do not cover the remaining ${(totalLinks - checkedLinks).toLocaleString('en-GB')} links.`, + ) + } + if ( + exists && + bodyStatus === 'complete' && + validation?.repeatedHashStable === null + ) { + caveats.push( + 'The repeated fetch was unavailable, so this run did not check body stability.', + ) + } + if (validation?.error) caveats.push(validation.error) + const dataStatus: LlmsAuditReport['dataStatus'] = validation?.error + ? validation.status + ? 'partial' + : 'unavailable' + : bodyStatus === 'unavailable' || + linkCheckStatus === 'partial' || + validation?.repeatedHashStable === null + ? 'partial' + : 'complete' if ( validation?.exists && @@ -163,7 +242,11 @@ function buildLlmsTxtAudit( evidence: { contentType: validation.contentType }, }) } - if (validation?.exists && validation.headingCount === 0) { + if ( + validation?.exists && + bodyStatus === 'complete' && + validation.headingCount === 0 + ) { issues.push({ id: 'llms-structure', severity: 'medium', @@ -207,17 +290,6 @@ function buildLlmsTxtAudit( evidence: { links: validation.invalidLinks }, }) } - if (validation?.linkLimitReached) { - issues.push({ - id: 'llms-link-limit', - severity: 'medium', - title: 'The link audit reached its limit', - plainEnglish: `The file contains ${validation.totalParsedLinks} parsed links. Only the first 100 were checked.`, - action: - 'Keep no more than 100 deliberate links so all file links can be checked.', - evidence: { parsedLinks: validation.totalParsedLinks, checkedLinks: 100 }, - }) - } const brokenLinks = validation?.links.filter( (link) => @@ -277,18 +349,6 @@ function buildLlmsTxtAudit( 'Generate the file during the build and remove timestamps, random ordering, or runtime rewriting.', }) } - if (validation?.oversized) { - issues.push({ - id: 'llms-file-size', - severity: 'medium', - title: 'llms.txt is larger than the review limit', - plainEnglish: `The file is ${validation.bytes ?? 'more than 100,000'} bytes. The v2 validator limit is 100,000 bytes.`, - action: - 'Keep the file under 100,000 bytes. Use a short list of useful entry points.', - evidence: { bytes: validation.bytes, limitBytes: 100_000 }, - }) - } - if (exists && pages.length < 3) { issues.push({ id: 'thin-llms-inventory', @@ -305,6 +365,7 @@ function buildLlmsTxtAudit( return { reportId: report.id, url: report.config.url, + dataStatus, exists, llmsTxtUrl: validation?.url ?? @@ -316,12 +377,34 @@ function buildLlmsTxtAudit( guidanceUrl: 'https://developers.google.com/search/updates#clarifying-guidance-on-llms-txt-files', headline: exists - ? validation - ? issues.length - ? `The optional llms.txt file was validated and ${issues.length} content or link issue${issues.length === 1 ? ' needs' : 's need'} review. It has no Google Search visibility impact.` - : `The optional llms.txt file was validated with ${validation.links.length} resolving link${validation.links.length === 1 ? '' : 's'} and a stable body. It has no Google Search visibility impact.` - : 'An optional llms.txt file is present, but its body was not validated in this crawl. It has no Google Search visibility impact.' - : 'No llms.txt file was found. This is not an SEO issue and requires no action.', + ? bodyStatus === 'unavailable' + ? 'The optional llms.txt file was found, but its body could not be checked within this audit. It has no Google Search visibility impact.' + : validation + ? issues.length + ? `The optional llms.txt file was validated and ${issues.length} content or link issue${issues.length === 1 ? ' needs' : 's need'} review. It has no Google Search visibility impact.` + : linkCheckStatus === 'partial' + ? `The optional llms.txt format was validated. ${checkedLinks} of ${totalLinks} links were checked, so the link evidence is partial. It has no Google Search visibility impact.` + : validation.repeatedHashStable === null + ? `The optional llms.txt format and ${checkedLinks} link${checkedLinks === 1 ? '' : 's'} were checked, but the repeat fetch was unavailable. It has no Google Search visibility impact.` + : `The optional llms.txt file was validated with ${checkedLinks} resolving link${checkedLinks === 1 ? '' : 's'} and a stable body. It has no Google Search visibility impact.` + : 'An optional llms.txt file is present, but its body was not validated in this crawl. It has no Google Search visibility impact.' + : validation?.error + ? 'The llms.txt audit could not determine whether the optional file exists. This has no Google Search visibility impact.' + : 'No llms.txt file was found. This is not an SEO issue and requires no action.', + caveats, + bodyEvidence: { + status: bodyStatus, + bytes: validation?.bytes, + bytesStatus: validation?.bytesStatus, + limitBytes: validation?.bodyLimitBytes, + limitExceeded: validation?.bodyLimitExceeded ?? false, + }, + linkCheck: { + status: linkCheckStatus, + checkedLinks, + totalLinks, + limit: linkCheckLimit, + }, issues, recommendedPages: pages.slice(0, 25).map((page) => ({ url: page.finalUrl, @@ -353,11 +436,24 @@ export function generateLlmsTxt( const maxUrls = Math.max( 1, Math.min( - Math.floor(options.maxUrls ?? MAX_LLMS_TXT_LINKS), - MAX_LLMS_TXT_LINKS, + Math.floor(options.maxUrls ?? DEFAULT_LLMS_TXT_MAX_URLS), + MAX_LLMS_TXT_URLS, + ), + ) + const tokenBudget = Math.max( + 1, + Math.min( + Math.floor(options.tokenBudget ?? DEFAULT_LLMS_TXT_TOKEN_BUDGET), + MAX_LLMS_TXT_TOKEN_BUDGET, + ), + ) + const maxBytes = Math.max( + MIN_GENERATED_LLMS_TXT_BYTES, + Math.min( + Math.floor(options.maxBytes ?? MAX_GENERATED_LLMS_TXT_BYTES), + MAX_GENERATED_LLMS_TXT_BYTES, ), ) - const tokenBudget = Math.max(1, Math.floor(options.tokenBudget ?? 12_000)) const normalizedTitle = truncateAtWord( options.title ?? originHost(report.config.url), 200, @@ -371,7 +467,8 @@ export function generateLlmsTxt( const description = normalizedDescription || `Curated entry points for agents reading ${originHost(report.config.url)}.` - const pages = candidatePages(report, options).slice(0, maxUrls) + const candidates = candidatePages(report, options) + const pages = candidates.slice(0, maxUrls) const sections = new Map() for (const page of pages) { const section = sectionForPage(page) @@ -381,6 +478,8 @@ export function generateLlmsTxt( const lines = [`# ${title}`, '', `> ${description}`, ''] let includedUrls = 0 const counts: Record = {} + let tokenBudgetReached = false + let byteBudgetReached = false for (const [sectionName, sectionPages] of sections) { const sectionLines = [`## ${sectionName}`, ''] @@ -395,9 +494,13 @@ export function generateLlmsTxt( ? `: ${truncateAtWord(page.metaDescription, 160)}` : '' const line = `- [${title}](${page.finalUrl})${note}` - const projected = [...lines, ...sectionLines, line, ''].join('\n') - if (Buffer.byteLength(projected) >= MAX_LLMS_TXT_BYTES) break + const projected = `${[...lines, ...sectionLines, line, ''].join('\n')}\n` + if (Buffer.byteLength(projected) > maxBytes) { + byteBudgetReached = true + break + } if (estimatedTokens(projected) > tokenBudget && includedUrls > 0) { + tokenBudgetReached = true break } sectionLines.push(line) @@ -412,11 +515,25 @@ export function generateLlmsTxt( } const content = `${lines.join('\n')}\n` + const reasons: GeneratedLlmsTxt['limits']['reasons'] = [] + if (candidates.length > maxUrls) reasons.push('maxUrls') + if (tokenBudgetReached || estimatedTokens(content) > tokenBudget) { + reasons.push('tokenBudget') + } + if (byteBudgetReached) reasons.push('maxBytes') return { content, includedUrls, estimatedTokens: estimatedTokens(content), sections: counts, + limits: { + candidateUrls: candidates.length, + maxUrls, + tokenBudget, + maxBytes, + truncated: reasons.length > 0, + reasons, + }, source: { reportId: report.id, status: report.status, diff --git a/packages/core/src/analyze/crawler/site-crawl-agent-discovery.test.ts b/packages/core/src/analyze/crawler/site-crawl-agent-discovery.test.ts index a149b5b1..8f797d39 100644 --- a/packages/core/src/analyze/crawler/site-crawl-agent-discovery.test.ts +++ b/packages/core/src/analyze/crawler/site-crawl-agent-discovery.test.ts @@ -47,10 +47,16 @@ const discovery: CrawlAgentDiscovery = { status: 200, contentType: 'text/plain', repeatedHashStable: true, + bodyDataStatus: 'complete', + bodyLimitBytes: 2_000_000, + bodyLimitExceeded: false, formatValid: true, formatErrors: [], headingCount: 1, totalParsedLinks: 0, + linkCheckStatus: 'complete', + linkCheckLimit: 100, + linksChecked: 0, linkLimitReached: false, links: [], invalidLinks: [], diff --git a/packages/mcp/src/crawler-llms-tools.ts b/packages/mcp/src/crawler-llms-tools.ts index cb5d6601..23e9b56d 100644 --- a/packages/mcp/src/crawler-llms-tools.ts +++ b/packages/mcp/src/crawler-llms-tools.ts @@ -5,6 +5,7 @@ import { generateLlmsTxt, latestCrawlReport, loadCrawlReport, + MAX_LLMS_TXT_URLS, } from '@seo/core' import * as z from 'zod/v4' import { assertExclusiveReportInput } from './crawler-tool-helpers.js' @@ -12,6 +13,9 @@ import * as crawlerInputs from './crawler-tool-inputs.js' import { fetchRateInput } from './fetch-rate.js' import { toolError, toolSuccess } from './tool-result.js' +const MCP_LLMS_TXT_MAX_BYTES = 64 * 1024 +const MCP_LLMS_TXT_MAX_TOKENS = 16_000 + export function registerCrawlerLlmsTools(server: McpServer): void { server.registerTool( 'seo_llms_txt_audit', @@ -78,8 +82,13 @@ export function registerCrawlerLlmsTools(server: McpServer): void { reportId: z.string().optional(), site: z.string().optional(), maxPages: crawlerInputs.crawlPageLimit, - maxUrls: z.number().int().min(1).max(100).optional(), - tokenBudget: z.number().int().positive().optional(), + maxUrls: z.number().int().min(1).max(MAX_LLMS_TXT_URLS).optional(), + tokenBudget: z + .number() + .int() + .min(1) + .max(MCP_LLMS_TXT_MAX_TOKENS) + .optional(), exclude: z.array(z.string()).optional(), title: z.string().optional(), description: z.string().optional(), @@ -125,6 +134,7 @@ export function registerCrawlerLlmsTools(server: McpServer): void { const generated = generateLlmsTxt(report, { maxUrls, tokenBudget, + maxBytes: MCP_LLMS_TXT_MAX_BYTES, exclude, title, description, diff --git a/packages/mcp/src/crawler-tools.test.ts b/packages/mcp/src/crawler-tools.test.ts index 414e3840..55992b49 100644 --- a/packages/mcp/src/crawler-tools.test.ts +++ b/packages/mcp/src/crawler-tools.test.ts @@ -57,6 +57,22 @@ function captureCrawlerTools(): Map { return tools } +test('llms MCP generation accepts more than 100 URLs and bounds agent output inputs', () => { + const tools = captureCrawlerTools() + const generate = tools.get('seo_llms_txt_generate') + assert.ok(generate) + const input = generate.config.inputSchema as Record< + string, + { safeParse(value: unknown): { success: boolean } } + > + + assert.equal(input.maxUrls?.safeParse(101).success, true) + assert.equal(input.maxUrls?.safeParse(10_000).success, true) + assert.equal(input.maxUrls?.safeParse(10_001).success, false) + assert.equal(input.tokenBudget?.safeParse(16_000).success, true) + assert.equal(input.tokenBudget?.safeParse(16_001).success, false) +}) + function mcpCrawlerKeySnapshot(result: JsonRecord) { const structured = result.structuredContent as JsonRecord const firstFix = firstRecord(structured.topFixes) diff --git a/packages/mcp/src/report-depth.ts b/packages/mcp/src/report-depth.ts index 55066b8c..1e776086 100644 --- a/packages/mcp/src/report-depth.ts +++ b/packages/mcp/src/report-depth.ts @@ -472,7 +472,7 @@ const REPORT_DEPTH_PRIMARY = { 'the source crawl date, cap, failures, and caveats', ], doNotClaim: [ - 'Reaching maxUrls or the token budget is possible truncation, not a complete inventory.', + 'Reaching maxUrls, the token budget, or the output byte budget is possible truncation, not a complete inventory.', 'A valid draft does not prove search or AI benefit, selection, indexing, or citations.', 'The crawler cannot determine publisher intent.', ],