From c3cd3e95f728841abb530f08b59482083f551103 Mon Sep 17 00:00:00 2001
From: Federico Bartoli <federico.bartoli@madisoft.it>
Date: Mon, 13 Apr 2026 19:38:36 +0200
Subject: [PATCH] Add current AI crawler user-agents to AI_AGENTS.crawlers
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Without these, the robots-txt checker can't tell when a site blocks
or only allows them.

- Claude-User, Claude-SearchBot — https://support.claude.com/en/articles/8896518
- OAI-SearchBot — https://platform.openai.com/docs/bots
- Applebot-Extended — https://support.apple.com/en-us/119829
- Meta-ExternalAgent, Meta-ExternalFetcher — https://developers.facebook.com/docs/sharing/bot/
- Perplexity-User — https://docs.perplexity.ai/guides/bots
---
 docs/checkers.md | 8 +++++---
 src/utils.js     | 7 +++++++
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/docs/checkers.md b/docs/checkers.md
index d8e67e6..409041b 100644
--- a/docs/checkers.md
+++ b/docs/checkers.md
@@ -15,10 +15,12 @@ Audits your `robots.txt` for AI agent accessibility.
 - AI crawlers explicitly allowed with `Allow: /` rules (+2)
 
 **Known AI crawlers checked:**
-- ClaudeBot, Claude-Web (Anthropic)
-- GPTBot, ChatGPT-User (OpenAI)
+- ClaudeBot, Claude-User, Claude-SearchBot, Claude-Web (Anthropic)
+- GPTBot, ChatGPT-User, OAI-SearchBot (OpenAI)
 - Google-Extended, GoogleOther (Google)
-- PerplexityBot (Perplexity)
+- Applebot-Extended (Apple)
+- Meta-ExternalAgent, Meta-ExternalFetcher (Meta)
+- PerplexityBot, Perplexity-User (Perplexity)
 - Amazonbot (Amazon)
 - cohere-ai (Cohere)
 - Bytespider (ByteDance)
diff --git a/src/utils.js b/src/utils.js
index f60fd54..75755b9 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -141,12 +141,19 @@ export async function loadConfig(dir) {
 export const AI_AGENTS = {
   crawlers: [
     { name: 'ClaudeBot', pattern: 'ClaudeBot' },
+    { name: 'Claude-User', pattern: 'Claude-User' },
+    { name: 'Claude-SearchBot', pattern: 'Claude-SearchBot' },
     { name: 'Claude-Web', pattern: 'Claude-Web' },
     { name: 'GPTBot', pattern: 'GPTBot' },
     { name: 'ChatGPT-User', pattern: 'ChatGPT-User' },
+    { name: 'OAI-SearchBot', pattern: 'OAI-SearchBot' },
     { name: 'Google-Extended', pattern: 'Google-Extended' },
     { name: 'GoogleOther', pattern: 'GoogleOther' },
+    { name: 'Applebot-Extended', pattern: 'Applebot-Extended' },
+    { name: 'Meta-ExternalAgent', pattern: 'Meta-ExternalAgent' },
+    { name: 'Meta-ExternalFetcher', pattern: 'Meta-ExternalFetcher' },
     { name: 'PerplexityBot', pattern: 'PerplexityBot' },
+    { name: 'Perplexity-User', pattern: 'Perplexity-User' },
     { name: 'Amazonbot', pattern: 'Amazonbot' },
     { name: 'cohere-ai', pattern: 'cohere-ai' },
     { name: 'Bytespider', pattern: 'Bytespider' },