diff --git a/packages/scoring/src/skills-map.test.ts b/packages/scoring/src/skills-map.test.ts new file mode 100644 index 0000000..f49acd3 --- /dev/null +++ b/packages/scoring/src/skills-map.test.ts @@ -0,0 +1,41 @@ +/** + * Every rule the engine can fail must have exactly one skill that teaches an agent how to fix it. + * `skills/rule-map.json` is that mapping; this test keeps it honest in both directions, so adding a + * rule without a skill (or renaming a skill directory) fails CI instead of leaving a dead end. + */ +import { existsSync, readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { technicalSeoRules } from './technical-seo-rules.js'; +import { aeoRules } from './aeo-rules.js'; +import { eeatSignalDefs } from './eeat-rules.js'; + +const SKILLS_DIR = fileURLToPath(new URL('../../../skills/', import.meta.url)); +const { rules: ruleMap } = JSON.parse(readFileSync(`${SKILLS_DIR}rule-map.json`, 'utf8')) as { + rules: Record; +}; +const ruleIds = [...technicalSeoRules, ...aeoRules, ...eeatSignalDefs].map((r) => r.id); + +describe('skills/rule-map.json', () => { + it('maps every rule id', () => { + expect(ruleIds.filter((id) => !(id in ruleMap))).toEqual([]); + }); + + it('maps no rule id that does not exist', () => { + expect(Object.keys(ruleMap).filter((id) => !ruleIds.includes(id))).toEqual([]); + }); + + it('points every rule at a skill that exists', () => { + const missing = [...new Set(Object.values(ruleMap))].filter( + (skill) => !existsSync(`${SKILLS_DIR}${skill}/SKILL.md`), + ); + expect(missing).toEqual([]); + }); + + // The router skill ships its own copy so it works when copied alone; it must not drift. + it('is mirrored exactly in the aeo router skill', () => { + const table = readFileSync(`${SKILLS_DIR}aeo/references/rules.md`, 'utf8'); + const rows = [...table.matchAll(/^\| `([^`]+)` \| \[`([^`]+)`\]/gm)].map(([, id, skill]) => [id, skill]); + expect(Object.fromEntries(rows)).toEqual(ruleMap); + }); +}); diff --git a/skills/README.md b/skills/README.md index 53f8b96..eed7e09 100644 --- a/skills/README.md +++ b/skills/README.md @@ -1,19 +1,62 @@ # AEO Toolkit skills -Claude Skills that turn the [`aeo-search` MCP server](https://aeo.advancelabs.dev/mcp) from a set -of tools into a set of **workflows**. +Claude Skills that give an agent the full method for Answer Engine Optimization: how to make a site +reachable, understandable and citable by ChatGPT, Claude, Perplexity, Gemini, Google AI Overviews +and Copilot, implemented in the stack the agent is actually working in. -The MCP server answers questions like "what are the numbers". These skills answer the -questions people actually ask: *why did traffic drop*, *which pages compete with each other*, -*what should I refresh*. The judgement in each skill — which readings are supportable, which -are guesses, when to say "I don't know" — is the part that does not fit in a tool description. +The MCP servers answer **"what is wrong, and how is it performing?"** These skills answer **"how +do I fix it here, and how do I know it worked?"** Each one follows the same loop: **detect** with +a rule or tool, **decide** whether it is really a problem, **implement** with per-stack recipes, +then **verify** with the same check. -| Skill | Answers | Tools used | +Start with [`aeo`](aeo/SKILL.md). It reads an audit, orders the work and hands each failing rule +to its owner. + +## Every audit rule has an owner + +The scoring engine has 58 rules. [`rule-map.json`](rule-map.json) assigns each one to exactly one +skill, and a test in `packages/scoring` fails CI if any of these drift: +- a rule exists without an owning skill +- a mapped skill does not exist +- the router's copy of the map diverges + +## The skills + +| Skill | Fixes | Uses | |---|---|---| -| [`seo-traffic-drop`](seo-traffic-drop/SKILL.md) | Traffic dropped. What caused it? | `gsc_traffic_drop`, `list_gsc_sites` | -| [`seo-cannibalization`](seo-cannibalization/SKILL.md) | Are my own pages competing? | `gsc_cannibalization`, `list_gsc_sites` | -| [`seo-content-decay`](seo-content-decay/SKILL.md) | What content needs refreshing? | `gsc_decay`, `gsc_traffic_drop`, `list_gsc_sites` | -| [`seo-high-intent-ctr`](seo-high-intent-ctr/SKILL.md) | Which buyer-intent searches do I rank for but lose the click on? | `gsc_search_analytics`, `gsc_ctr_gaps`, `list_gsc_sites` | +| [`aeo`](aeo/SKILL.md) | Router: audit → triage → hand off | `analyze_website_aeo` | +| **Access** | | | +| [`aeo-ai-crawler-access`](aeo-ai-crawler-access/SKILL.md) | AI bots blocked by robots.txt or a CDN; training vs search bots | `analyze_website_aeo` | +| [`aeo-server-rendering`](aeo-server-rendering/SKILL.md) | Content that only exists after JavaScript runs | `analyze_website_aeo` | +| [`aeo-llms-txt`](aeo-llms-txt/SKILL.md) | llms.txt, served without cannibalizing your pages | `analyze_website_aeo` | +| [`aeo-indexability`](aeo-indexability/SKILL.md) | Sitemaps, canonicals, noindex, redirects, hreflang, HTTPS | `gsc_url_inspect`, `engine_divergence` | +| **Identity** | | | +| [`aeo-organization-entity`](aeo-organization-entity/SKILL.md) | Organization JSON-LD, `@id` graph, consistent identity | `analyze_website_aeo` | +| [`aeo-article-author-schema`](aeo-article-author-schema/SKILL.md) | Article + Person author markup | `analyze_website_aeo` | +| [`aeo-faq-qa-schema`](aeo-faq-qa-schema/SKILL.md) | FAQPage / QAPage / HowTo, honestly scoped | `analyze_website_aeo` | +| [`aeo-structured-data-validation`](aeo-structured-data-validation/SKILL.md) | Invalid or conflicting structured data | `analyze_website_aeo` | +| [`aeo-product-schema`](aeo-product-schema/SKILL.md) | Product / Offer / Review for stores | `analyze_website_aeo` | +| [`aeo-local-business-schema`](aeo-local-business-schema/SKILL.md) | LocalBusiness, NAP consistency, Business Profile | `check_ai_visibility` | +| **Extractability** | | | +| [`aeo-answer-first-content`](aeo-answer-first-content/SKILL.md) | Question headings, answer-first paragraphs, depth | `discover_ranking_prompts` | +| [`aeo-site-structure`](aeo-site-structure/SKILL.md) | Breadcrumbs and internal linking | `analyze_website_aeo` | +| [`aeo-page-metadata`](aeo-page-metadata/SKILL.md) | Titles, descriptions, Open Graph, lang, viewport, alt text | `gsc_ctr_gaps` | +| [`aeo-content-freshness`](aeo-content-freshness/SKILL.md) | Honest dates across page, JSON-LD and sitemap | `gsc_decay` | +| [`aeo-eeat-trust-signals`](aeo-eeat-trust-signals/SKILL.md) | About, contact, policies, references | `analyze_website_aeo` | +| **Off-site** | | | +| [`aeo-citation-earning`](aeo-citation-earning/SKILL.md) | Where competitors get cited, mentions, outreach | backlink tools | +| **Measurement** | | | +| [`aeo-visibility-tracking`](aeo-visibility-tracking/SKILL.md) | Prompt sets, baselines, Perplexity citation share | visibility tools | +| [`aeo-ai-referral-analytics`](aeo-ai-referral-analytics/SKILL.md) | AI-assistant referral traffic in GA4 | `ga4_run_report` | +| [`seo-bing-coverage`](seo-bing-coverage/SKILL.md) | Google vs Bing gaps | `engine_divergence`, `bing_*` | +| [`seo-traffic-drop`](seo-traffic-drop/SKILL.md) | Traffic dropped. What caused it? | `gsc_traffic_drop` | +| [`seo-cannibalization`](seo-cannibalization/SKILL.md) | Are my own pages competing? | `gsc_cannibalization` | +| [`seo-content-decay`](seo-content-decay/SKILL.md) | What content needs refreshing? | `gsc_decay` | +| [`seo-high-intent-ctr`](seo-high-intent-ctr/SKILL.md) | Buyer-intent searches that rank but lose the click | `gsc_ctr_gaps` | + +Implementation recipes live in each skill's `references/` (Next.js, Astro, WordPress, static HTML, +and Shopify/WooCommerce where relevant). The agent only loads the one for the stack in front of +it. ## Install @@ -22,7 +65,7 @@ Skills are plain directories. Copy the ones you want into your skills folder: ```bash # Claude Code, for one project mkdir -p .claude/skills -cp -r path/to/aeo-toolkit/skills/seo-traffic-drop .claude/skills/ +cp -r path/to/aeo-toolkit/skills/aeo* .claude/skills/ # or for every project cp -r path/to/aeo-toolkit/skills/* ~/.claude/skills/ @@ -31,15 +74,20 @@ cp -r path/to/aeo-toolkit/skills/* ~/.claude/skills/ They activate on their own when a request matches the `description` in the frontmatter. You do not need to name the skill. -## They need the MCP server +## MCP servers: helpful, mostly optional -A skill is instructions, not data. Every one of these calls the `aeo-search` MCP server, which -holds the Google OAuth connection to your Search Console account. Without it the skill has -nothing to read. +The `aeo-*` skills work without any MCP server. Each **Detect** step has a no-MCP fallback, for +example `curl` or view-source. Connecting the servers makes detection and verification faster +and more complete. The `seo-*` and measurement skills read Search Console, GA4, Bing or +Perplexity data, and they need the matching server. -Connect it from [aeo.advancelabs.dev/mcp](https://aeo.advancelabs.dev/mcp). You will need -Search Console access for the property you want to analyse; the server handles the OAuth flow -and stores tokens encrypted. +| Server | Endpoint | Auth | +|---|---|---| +| `aeo-visibility` | `https://aeo.advancelabs.dev/api/mcp/ai-visibility/mcp` | none | +| `aeo-backlink` | `https://aeo.advancelabs.dev/api/mcp/backlink/mcp` | none | +| `aeo-search` | `https://aeo.advancelabs.dev/api/mcp/search/mcp` | Google sign-in; Bing key optional | + +Setup for each client is at [advancelabs.dev/mcp](https://advancelabs.dev/mcp). > **On "no API keys" claims.** Search Console requires OAuth. Anything promising GSC data with > no authentication is either reading an already-authenticated local session or not reading @@ -47,19 +95,28 @@ and stores tokens encrypted. ## What these will not do -- **They cannot see your competitors' data.** Search Console reports your property only. Any - competitor comparison needs a third-party rank-tracking source, which these do not use. -- **They cannot separate seasonality from decline.** GSC has no notion of why demand moved. - Each skill says so where it matters rather than implying more certainty than the data - supports. -- **They do not change anything.** Every tool is read-only. +- **They don't promise citations.** A fix removes a known blocker. Whether an engine then cites + you is measured, not assumed. +- **The visibility tools see one engine.** `check_ai_visibility` and its siblings query + Perplexity, so their readings are Perplexity citations, not "AI" in general. +- **They can't see competitors' Search Console data.** Competitor comparisons use public + signals (citations, links) only. +- **They don't change anything remotely.** Every MCP tool is read-only. Code changes happen in + your repo, where you can review them. ## Contributing a skill -Same bar as the rest of the repo: a skill should encode judgement a tool description cannot, -be explicit about what the data cannot answer, and never assert a cause it cannot evidence. +Same bar as the rest of the repo: +- encode judgement a tool description cannot hold +- be explicit about what the data cannot answer +- never assert a cause you can't evidence +- date every volatile fact (bot names, Google policies) in a `## Sources` section, against a + primary source + +The frontmatter `description` decides whether the skill activates, so write it as *when to use +this* (including the phrase "Use when"), not as a title. -The frontmatter `description` is what decides whether the skill activates, so write it as -*when to use this*, not as a title. A test in the private platform repo reads these skills, validates the -frontmatter and checks that every tool a skill mentions is registered on the server, so a -renamed tool fails CI there instead of silently breaking the skill. +Two tests keep the skills honest: +- `packages/scoring/src/skills-map.test.ts` here checks rule ownership. +- A test in the private platform repo checks frontmatter, and checks that every MCP tool a skill + names is registered on one of the three servers. diff --git a/skills/aeo-ai-crawler-access/SKILL.md b/skills/aeo-ai-crawler-access/SKILL.md new file mode 100644 index 0000000..1c0accd --- /dev/null +++ b/skills/aeo-ai-crawler-access/SKILL.md @@ -0,0 +1,199 @@ +--- +name: aeo-ai-crawler-access +description: Check and fix whether AI crawlers can reach a site, and decide which ones to allow. Use when someone asks "why isn't ChatGPT or Perplexity citing my site", "should I block GPTBot", "allow AI crawlers", "AI bots blocked", "robots.txt for AI", "Cloudflare is blocking AI bots", or when the audit fails aeo.ai-bots-allowed or tech.robots-present. Covers training bots vs search and user-fetch bots, robots.txt, and CDN/WAF blocking that overrides robots.txt. Uses the aeo-visibility MCP server if connected; works without it. +--- + +# AI crawler access + +Answer engines can only cite what their bots can fetch. This skill checks robots.txt and the +CDN in front of it, then helps choose which AI bots to allow. Blocking AI *training* while +allowing AI *search* is a legitimate choice, and the audit rules do not always know that. + +## When this matters / when it doesn't + +It matters when a site wants to appear in ChatGPT search, Claude, Perplexity, Gemini or +Google AI Overviews and has never looked at its robots.txt or its CDN bot settings. + +It does **not** matter much, and the finding may be a deliberate choice, when: + +- The owner has decided not to be used for model training. Blocking `GPTBot` or `ClaudeBot` + is a policy decision, not a defect. **Ask** before recommending they undo it. +- The site is private, paywalled or intentionally unlisted. +- Only the training bots are blocked and the search/retrieval bots are allowed. That site + can still be cited. The audit rule may still fail it (see below). Say so plainly. + +## Detect + +Rules owned: + +- `tech.robots-present`: passes when `/robots.txt` answers 2xx with a non-HTML content type. + A 404, or a catch-all that returns the app shell as `text/html`, fails. +- `aeo.ai-bots-allowed`: fails when robots.txt disallows `/` for any of **GPTBot, ClaudeBot, + PerplexityBot, OAI-SearchBot or Google-Extended**. No robots.txt means nothing is blocked + and the rule passes. Note the list mixes training bots (GPTBot, ClaudeBot, Google-Extended) + with search bots (OAI-SearchBot, PerplexityBot). A site that blocks only the training bots on + purpose will fail this rule. Report it as "intentional, rule is blunt", not as a bug. + +With the MCP server: call `analyze_website_aeo` on the site root. The result lists AI-bot +access per bot. + +Without it: + +```bash +curl -sI https://example.com/robots.txt | head -5 # 200 and not text/html +curl -s https://example.com/robots.txt +# Does the CDN treat bots differently from robots.txt? Compare status codes: +curl -s -o /dev/null -w "%{http_code}\n" -A "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.4; +https://openai.com/gptbot" https://example.com/ +curl -s -o /dev/null -w "%{http_code}\n" -A "Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)" https://example.com/ +``` + +A 403, 429 or a challenge page for a bot while a normal browser gets 200 means something in +front of the site is blocking it, whatever robots.txt says. Caveat: a CDN that verifies bots +by IP range will let the real bot through and block your spoofed curl, so a `200` here proves +little and a `403` is a hint, not proof. Server logs and the CDN's bot analytics are the real +evidence. + +## Decide + +Each vendor documents separate user agents for separate jobs. Verified against their own +pages on 2026-11-13. + +| Operator | User agent | Job | Blocking it means | +|---|---|---|---| +| OpenAI | `OAI-SearchBot` | Surfaces sites in ChatGPT search | Not shown in ChatGPT search answers (can still appear as plain links) | +| OpenAI | `GPTBot` | Crawls content that may train foundation models | Opt-out of training | +| OpenAI | `ChatGPT-User` | Fetches a page when a user asks ChatGPT or a Custom GPT | OpenAI says robots.txt rules may not apply to it | +| OpenAI | `OAI-AdsBot` | Checks landing pages of ads submitted to ChatGPT | Only matters if you advertise there | +| Anthropic | `ClaudeBot` | Collects content that may contribute to training | Opt-out of training | +| Anthropic | `Claude-SearchBot` | Indexes content to improve search results | May reduce visibility in Claude search answers | +| Anthropic | `Claude-User` | Fetches when a user asks Claude a question | May reduce visibility for user-directed web search | +| Perplexity | `PerplexityBot` | Surfaces sites in Perplexity search; not used for foundation-model training | Not shown in Perplexity results | +| Perplexity | `Perplexity-User` | Fetches when a user asks | Perplexity says it generally ignores robots.txt | +| Google | `Googlebot` | Search, and therefore AI Overviews and AI Mode | Leaves Search entirely | +| Google | `Google-Extended` | Product token only, no separate fetch. Controls use for Gemini training and for grounding in Gemini Apps / Vertex | Does not affect Search inclusion or ranking | +| Apple | `Applebot` | Powers Spotlight, Siri, Safari search; data may also train Apple models | Leaves Apple search | +| Apple | `Applebot-Extended` | Opt-out token for training Apple foundation models | Opt-out of training; Applebot can still crawl | + +Things this table settles that people get wrong: + +- **There is no robots.txt switch for Google AI Overviews.** Google says `Googlebot` is the + control, and `nosnippet`, `data-nosnippet`, `max-snippet` or `noindex` limit what is shown. + `Google-Extended` does not remove you from AI Overviews. +- **User-fetch bots (`ChatGPT-User`, `Perplexity-User`, `Claude-User`) act for a person.** + OpenAI and Perplexity both say robots.txt may not govern them. Do not expect a `Disallow` + to stop them, and do not call a hit from one a violation. +- OpenAI and Perplexity both say robots.txt changes can take up to about 24 hours to take effect. Anthropic gives no number. + +Recommended defaults, offered as options, not orders: + +1. **Wants to be cited** (most sites): allow all the search and user-fetch bots. Training bots are + a separate choice for the owner. +2. **Wants no training but wants citations**: block `GPTBot`, `ClaudeBot`, `Google-Extended`, + `Applebot-Extended` (and optionally `CCBot`, Common Crawl's crawler; its docs were not + re-checked for this skill). Allow `OAI-SearchBot`, `Claude-SearchBot`, `PerplexityBot`, + `Googlebot`, `Bingbot`. Expect `aeo.ai-bots-allowed` to fail for this site and say that the + failure is the chosen trade-off. +3. **Wants out of AI entirely**: block everything above, and understand that this removes the + site from AI citations. Confirm that is the intent. + +Anthropic also says that since 2026-10-30 it honours a machine-readable "no AI training" +preference signal based on the IETF aipref draft, and that a robots.txt rule naming `ClaudeBot` +takes priority over the general signal. Google says it intends to support the same shared +signal in early 2027. The standard is still a draft, so treat it as extra, not a replacement +for naming the bots. + +## Implement + +Shape of the fix, any stack: + +1. Serve a real `/robots.txt` (text/plain, 200). Reference the sitemap. +2. Write explicit groups for the bots the decision above names. A group naming a bot replaces + the `User-agent: *` group for that bot, so repeat any `Disallow` paths you still want. +3. Check the layer in front of the origin (CDN, WAF, host "bot protection", plugins). + +``` +User-agent: * +Allow: / + +# Allowed: search and retrieval +User-agent: OAI-SearchBot +Allow: / +User-agent: Claude-SearchBot +Allow: / +User-agent: PerplexityBot +Allow: / + +# Opt out of training only (delete this block if the owner is happy to be trained on) +User-agent: GPTBot +Disallow: / +User-agent: ClaudeBot +Disallow: / +User-agent: Google-Extended +Disallow: / +User-agent: Applebot-Extended +Disallow: / + +Sitemap: https://example.com/sitemap.xml +``` + +**The CDN overrides robots.txt.** robots.txt is a request; a WAF rule is enforcement. If a bot +gets a 403 or a challenge at the edge it never reads robots.txt. Anthropic says as much: +blocking its IP ranges can stop it reading robots.txt at all, so an opt-out should be done in +robots.txt, not by IP. Common culprits: + +- **Cloudflare managed robots.txt / AI Crawl Control.** Setting the AI *Training* policy to + Disallow adds managed `Disallow` lines (Amazonbot, Applebot-Extended, Bytespider, CCBot, + ClaudeBot, Google-Extended, GPTBot, meta-externalagent) and, with Bot Preference Sync on, + enables blocking at the edge for AI training crawlers. Cloudflare says mixed-use crawlers are + excluded from the edge block, and the Search and Agent policies add their own directives when + set to Block. If the site has no robots.txt, Cloudflare serves one with those rules. Check + Security settings, Bot traffic, "AI Crawler and Agent access policies". +- A custom WAF rule, rate limit or "Bot Fight Mode" that challenges unknown user agents. +- Host-level firewall plugins (WordPress security plugins are the usual source). + +If you want a bot through a WAF, allow it by **user agent plus published IP range**, not user +agent alone, because user agents are trivially forged. OpenAI publishes ranges at +`openai.com/searchbot.json`, `gptbot.json`, `chatgpt-user.json`; Perplexity at +`perplexity.ai/perplexitybot.json` and `perplexity-user.json`; Anthropic links its list from +the crawler help page. Perplexity's guide walks through a Cloudflare Skip rule ordered above any +blocking rule. + +Per-stack file recipes: read `references/nextjs.md`, `references/astro.md`, +`references/wordpress.md` or `references/static-html.md`. + +## Verify + +1. `curl -s https://example.com/robots.txt` shows the groups you intended, as text/plain. +2. Re-run `analyze_website_aeo`. `tech.robots-present` should pass. `aeo.ai-bots-allowed` + should pass if you allowed all five key bots; if you chose option 2 or 3 it will fail + by design, and the report should say which bots are blocked on purpose. +3. Check the edge: look in the CDN's bot analytics or server logs for 403/429 on the search + bots' user agents over the last few days. No blocks there is the real pass. +4. Allow a day for the bots to pick up the change. + +## Pitfalls + +- Blocking `Google-Extended` and expecting to leave AI Overviews. It does not. +- Adding `User-agent: *` / `Disallow: /` "temporarily" on a staging config that ships to + production. Check the live file after every deploy. +- A catch-all route that returns the app shell for `/robots.txt`. The audit (and some + crawlers) treat an HTML response as "no robots.txt". +- Allowing a bot in robots.txt and leaving a Cloudflare AI policy on Block. Both layers must agree. +- Treating `ChatGPT-User` or `Perplexity-User` traffic as crawling. It is a person asking. +- `Disallow:` with an empty value allows everything; `Disallow: /` blocks everything. One + character, opposite meanings. +- Legacy names (`Claude-Web`, `anthropic-ai`) still appear in some lists and the toolkit still probes + them, but Anthropic's current help page lists only ClaudeBot, Claude-User and Claude-SearchBot. +- Naming a bot in a group with no rules does nothing useful. Give it `Allow: /` or `Disallow: /`. + +## Sources + +- OpenAI crawlers: https://platform.openai.com/docs/bots (verified 2026-11-13) +- Anthropic crawlers: https://support.claude.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler (verified 2026-11-13) +- Perplexity crawlers and WAF guide: https://docs.perplexity.ai/guides/bots (verified 2026-11-13) +- Google common crawlers (Google-Extended): https://developers.google.com/search/docs/crawling-indexing/google-common-crawlers (verified 2026-11-13) +- Google AI features and your website: https://developers.google.com/search/docs/appearance/ai-features (verified 2026-11-13) +- Apple Applebot: https://support.apple.com/en-us/119829 (verified 2026-11-13) +- Cloudflare managed robots.txt: https://developers.cloudflare.com/bots/additional-configurations/managed-robots-txt/ (verified 2026-11-13) +- robots.txt introduction (Google): https://developers.google.com/search/docs/crawling-indexing/robots/intro (verified 2026-11-13) +- Common Crawl's CCBot page was not fetched; its role is stated from general knowledge and not verified. diff --git a/skills/aeo-ai-crawler-access/references/astro.md b/skills/aeo-ai-crawler-access/references/astro.md new file mode 100644 index 0000000..f5f1bf0 --- /dev/null +++ b/skills/aeo-ai-crawler-access/references/astro.md @@ -0,0 +1,43 @@ +# Astro 5 + +Static file: put `robots.txt` in `public/`. For a dynamic file (to build the sitemap URL from +`site`), use an endpoint. Astro calls `GET` at build time in static mode and writes the body. + +```ts +// src/pages/robots.txt.ts +import type { APIRoute } from 'astro' + +export const GET: APIRoute = ({ site }) => { + const sitemap = new URL('sitemap-index.xml', site) + const body = `User-agent: * +Allow: / + +User-agent: OAI-SearchBot +Allow: / +User-agent: Claude-SearchBot +Allow: / +User-agent: PerplexityBot +Allow: / + +# Training opt-out. Remove if the owner is happy to be trained on. +User-agent: GPTBot +Disallow: / +User-agent: ClaudeBot +Disallow: / +User-agent: Google-Extended +Disallow: / +User-agent: Applebot-Extended +Disallow: / + +Sitemap: ${sitemap.href} +` + return new Response(body, { headers: { 'Content-Type': 'text/plain; charset=utf-8' } }) +} +``` + +- `site` must be set in `astro.config.mjs`, otherwise `site` is undefined here and the + `@astrojs/sitemap` integration refuses to build. The integration writes `sitemap-index.xml` + and `sitemap-0.xml`. +- Do not keep both `public/robots.txt` and `src/pages/robots.txt.ts`. +- If the host is Cloudflare Pages/Workers, check the AI Crawl Control policy as described in + the main skill. It can add its own `Disallow` lines on top of yours. diff --git a/skills/aeo-ai-crawler-access/references/nextjs.md b/skills/aeo-ai-crawler-access/references/nextjs.md new file mode 100644 index 0000000..a9701cc --- /dev/null +++ b/skills/aeo-ai-crawler-access/references/nextjs.md @@ -0,0 +1,31 @@ +# Next.js (App Router) + +`app/robots.ts` returns a `MetadataRoute.Robots` object and Next serves it as `/robots.txt`. +Multiple bots go in the `rules` array; `userAgent` can be a string or an array. + +```ts +// app/robots.ts +import type { MetadataRoute } from 'next' + +export default function robots(): MetadataRoute.Robots { + return { + rules: [ + { userAgent: '*', allow: '/' }, + { userAgent: ['OAI-SearchBot', 'Claude-SearchBot', 'PerplexityBot'], allow: '/' }, + // Training opt-out. Remove this entry if the owner is happy to be trained on. + { userAgent: ['GPTBot', 'ClaudeBot', 'Google-Extended', 'Applebot-Extended'], disallow: '/' }, + ], + sitemap: 'https://example.com/sitemap.xml', + } +} +``` + +- A static `public/robots.txt` also works, but do not have both: pick one. +- Use the real production origin in `sitemap`. Do not build it from a preview URL env var, or + the deploy preview's host ends up in production's robots.txt. +- Check `middleware.ts` / `proxy.ts` matchers: a rule that rewrites or redirects unknown user + agents will also catch bots. Exclude `/robots.txt` and `/sitemap.xml` from auth middleware. +- On Vercel, also check Firewall rules and Attack Challenge Mode, which challenge bots at the + edge regardless of robots.txt. + +Verify: `curl -s https:///robots.txt`. diff --git a/skills/aeo-ai-crawler-access/references/static-html.md b/skills/aeo-ai-crawler-access/references/static-html.md new file mode 100644 index 0000000..13df975 --- /dev/null +++ b/skills/aeo-ai-crawler-access/references/static-html.md @@ -0,0 +1,12 @@ +# Static HTML / any host + +1. Create `robots.txt` in the directory the host serves as the site root (`public/`, `dist/`, + `docs/`, whatever deploys to `/`). Use the template in the main skill. +2. Make sure the host serves it as `text/plain` and does not rewrite unknown paths to + `index.html` for it. A single-page-app fallback rule that catches `/robots.txt` makes it + `text/html`, and the audit then says it is missing. +3. Per-subdomain: each host needs its own robots.txt (`www.example.com`, `blog.example.com`). + Anthropic's page says to do this for every subdomain you want covered. +4. If the host or CDN has an "AI bots" toggle (Cloudflare has one), check it matches the robots.txt you just wrote. + +Verify: `curl -sI https://example.com/robots.txt` shows `200` and `content-type: text/plain`. diff --git a/skills/aeo-ai-crawler-access/references/wordpress.md b/skills/aeo-ai-crawler-access/references/wordpress.md new file mode 100644 index 0000000..784b8f7 --- /dev/null +++ b/skills/aeo-ai-crawler-access/references/wordpress.md @@ -0,0 +1,33 @@ +# WordPress 6.x + +WordPress serves a **virtual** `/robots.txt` unless a physical `robots.txt` exists in the web +root. A physical file wins and ignores everything below. + +Add rules through the `robots_txt` filter (parameters: `$output`, `$public`): + +```php +// in a small mu-plugin or the theme's functions.php +add_filter( 'robots_txt', function ( $output, $public ) { + if ( ! $public ) { + return $output; // site set to "discourage search engines": leave it alone + } + $output .= "\nUser-agent: OAI-SearchBot\nAllow: /\n"; + $output .= "User-agent: Claude-SearchBot\nAllow: /\n"; + $output .= "User-agent: PerplexityBot\nAllow: /\n"; + // Training opt-out. Remove if the owner is happy to be trained on. + foreach ( array( 'GPTBot', 'ClaudeBot', 'Google-Extended', 'Applebot-Extended' ) as $bot ) { + $output .= "\nUser-agent: {$bot}\nDisallow: /\n"; + } + return $output; +}, 20, 2 ); +``` + +- Settings, Reading, "Discourage search engines from indexing this site" makes WordPress emit + `Disallow: /` for everyone. This is the most common reason a WordPress site is invisible. + It is often left on after a staging copy goes live. +- Yoast, Rank Math and AIOSEO each edit robots.txt from their own screen. Look there first; + two plugins writing it produces a file nobody chose. +- Security plugins and host firewalls (Wordfence, Sucuri, host-level bot protection) can block + AI user agents before WordPress runs. Check their logs, not robots.txt. +- Core generates a sitemap at `/wp-sitemap.xml`; SEO plugins replace it with their own. + Reference whichever one is live. diff --git a/skills/aeo-ai-referral-analytics/SKILL.md b/skills/aeo-ai-referral-analytics/SKILL.md new file mode 100644 index 0000000..a6b4392 --- /dev/null +++ b/skills/aeo-ai-referral-analytics/SKILL.md @@ -0,0 +1,89 @@ +--- +name: aeo-ai-referral-analytics +description: Measure traffic that AI assistants send to a site, using Google Analytics 4. Use when someone asks "how much traffic comes from ChatGPT/Perplexity/Claude", "are AI answers sending us visitors", "set up an AI traffic channel in GA4", "which pages do AI assistants send people to", or "is our AEO work driving visits". Covers the GA4 query, a reusable referrer regex, landing pages, and why the number is always an undercount. Requires the aeo-toolkit aeo-search MCP server with Google OAuth. +--- + +# AI referral analytics (GA4) + +AI assistants send some visitors with a referrer or a `utm_source` tag, and GA4 records +them. That subset is measurable. It is **a floor, not the total**: most AI influence never +produces a tagged click. Report it as "at least N sessions", never as the size of AI impact. + +## When this matters / when it doesn't + +- Matters: showing whether AEO work produces visits, and which pages earn them. +- Weak evidence on its own: small sites will have tens of sessions, where week-to-week + moves are noise. Say so instead of charting a trend. +- Not a visibility metric. Whether the site is *cited* is `aeo-visibility-tracking`. + +## Steps + +1. `list_ga4_properties`, confirm the property with the user. Don't guess the ID. +2. **Pull sources, then filter yourself.** `ga4_run_report` takes `propertyId`, `dateRanges`, + `dimensions`, `metrics` and `limit` only: **there is no filter argument.** So request + a broad grouping and filter the returned rows: + - dimensions: `sessionSource`, `sessionMedium` (add `landingPage` for step 4) + - metrics: `sessions`, `engagedSessions`, `keyEvents` (use only metric names GA4 accepts for + the property; if a call errors, drop the metric rather than guessing) + - `limit`: high (e.g. 10000) so the long tail of sources is not truncated. + Use a range of at least 28 days. Keep the same length when comparing periods. +3. **Match AI sources.** Apply the regex in `references/ga4-setup.md` to `sessionSource`. + Also list sources that *almost* match and any unfamiliar source containing `ai`, `gpt`, + `chat` or `claude`, and show them to the user: new assistants appear faster than any + list. Do not silently drop or add rows. +4. **Landing pages.** Re-run with `landingPage` and `sessionSource`, filter to the AI + sources, rank pages by sessions. These are the pages assistants actually send people to. + Compare against pages you expected; mismatches are the useful finding. +5. **Quality.** Compare engaged sessions per session and key events for AI sessions against + organic search. With under ~30 AI sessions, say the comparison is too thin to read. +6. **Trend.** Re-run `date` + `sessionSource` weekly if the volume supports it. + +## How the traffic is tagged + +- OpenAI's publisher FAQ states ChatGPT adds `utm_source=chatgpt.com` to links it sends, and + that this tracking depends on the site allowing OAI-SearchBot. Verified via an excerpt + of that FAQ; OpenAI's help pages block automated fetching, so re-read it before quoting. +- For other assistants (Perplexity, Claude, Gemini, Copilot) we did **not** verify referrer + hostnames or tagging against primary sources. The regex lists likely hostnames; step 3's + "unfamiliar source" review is how you confirm what really appears in this property. +- GA4 dimension names `sessionSource`, `sessionMedium`, `sessionDefaultChannelGroup`, + `landingPage` and `pageReferrer` exist in the Data API schema (verified 2026-11-13). + `pageReferrer` is the full previous URL, per page, not per session. + +## How to read the result + +- **Signal:** AI sessions that are consistently present across weeks, concentrated on + identifiable pages, with engagement similar to other referral traffic. +- **Noise:** week-to-week swings on a few dozen sessions; a one-day spike (often a single + mention or a crawler/bot). +- Always state the denominator: AI sessions as a share of all sessions, with both counts. + +## What it cannot tell you (undercounting, be explicit) + +- **No-click influence.** A user reads an answer that names the brand, then searches it + or types the URL later. That arrives as organic or **direct**. +- **Stripped or missing referrers.** Copied links, apps, in-app browsers and some privacy + settings drop the referrer, so visits land in "direct". One secondary source says this + happens for in-app browsers and copied links; not confirmed by OpenAI. +- **Assistants that don't tag.** Without a referrer or UTM, nothing in GA4 identifies them. +- **Bot and crawler traffic** is not a visitor; GA4 filters known bots, not all of them, so crawler + activity needs server logs, not GA4. +- Causality: a visit does not show that an AEO change caused it. + +Do not "correct" for undercounting with a multiplier. There is no verified one. + +## Pitfalls + +- Presenting the AI-referral total as total AI impact. +- Reading `direct` growth as AI traffic with no other evidence. +- A regex that is stale: review unmatched sources each time. +- Comparing periods of different length or weekday mix. +- Changing the channel definition mid-trend without noting the date. + +## Sources + +- GA4 Data API schema, https://developers.google.com/analytics/devguides/reporting/data/v1/api-schema (dimension names), verified 2026-11-13. +- OpenAI Publishers and Developers FAQ, help.openai.com article 12627856 (`utm_source=chatgpt.com`), verified 2026-11-13 via search excerpt only. +- OpenAI crawler docs, https://platform.openai.com/docs/bots, verified 2026-11-13. +- Tool behaviour from the aeo-search `ga4_run_report` definition, verified 2026-11-13. +- Hostnames other than chatgpt.com: NOT verified. diff --git a/skills/aeo-ai-referral-analytics/references/ga4-setup.md b/skills/aeo-ai-referral-analytics/references/ga4-setup.md new file mode 100644 index 0000000..b5cb1d5 --- /dev/null +++ b/skills/aeo-ai-referral-analytics/references/ga4-setup.md @@ -0,0 +1,39 @@ +# GA4 setup for AI referral traffic + +## Regex for AI sources (Session source) + +Starting point, case-insensitive match on `sessionSource`: + +``` +(?i)^(chatgpt\.com|chat\.openai\.com|perplexity(\.ai)?|claude\.ai|gemini\.google\.com|copilot\.microsoft\.com|copilot|edgeservices\.bing\.com)$ +``` + +Only `chatgpt.com` (via OpenAI's documented `utm_source`) is verified against a primary source. +The rest are commonly observed hostnames to confirm in your own data. Review unmatched +sources before trusting totals, and extend the regex when you see a new assistant. +Anchor with `^...$` so `gemini.google.com` does not catch ordinary `google`. + +GA4's "matches regex" is a full match, so the anchors are optional there; keep them for +filtering returned rows in code. + +## Custom channel group (GA4 UI) + +Whether a new channel group applies to past data is a GA4 behaviour we did not verify; test it on a past range before promising history. + +1. Admin > Data display > Channel groups > Create new channel group. +2. Add a channel named "AI assistants". +3. Condition: Source, "matches regex", paste the regex above. +4. Drag it **above** "Referral" (channels match top to bottom; the first match wins). +5. Save. Use the new group in Acquisition reports and as a comparison. + +## Exploration (no channel group needed) + +Explore > Free form. Dimensions: Session source, Landing page. Metrics: Sessions, Engaged +sessions, Key events. Filter: Session source matches regex (above). + +## Checks + +- Compare totals from the API (`sessionSource`, filtered in code) with the Explore view; + they should agree for the same range. +- Look at Session medium for the matched sources. `(none)` suggests missing referrer; + `referral` suggests a referrer was passed. The mix is informative, not a verdict. diff --git a/skills/aeo-answer-first-content/SKILL.md b/skills/aeo-answer-first-content/SKILL.md new file mode 100644 index 0000000..51e981c --- /dev/null +++ b/skills/aeo-answer-first-content/SKILL.md @@ -0,0 +1,147 @@ +--- +name: aeo-answer-first-content +description: Restructure existing pages so an answer engine can lift a clean answer out of them, without wrecking the author's voice. Use when someone asks why ChatGPT, Perplexity or Google AI Overviews don't quote their page, how to write for AI answers, how to structure a post for featured snippets, whether to add FAQ-style or question headings, why a page is flagged as thin, has no headings, has two H1s or skips heading levels, or when an audit fails aeo.question-headings, aeo.answerable-content, aeo.content-extractability, tech.content-not-thin, tech.single-h1 or tech.heading-hierarchy. Uses the aeo-visibility MCP server if connected; works without it. +--- + +# Answer-first content + +An answer engine can only quote what it can find and cut out cleanly. Pages that open each section +with the direct answer, then explain, are easier to quote than pages that circle the point for four +paragraphs. This skill covers how to restructure a page that way, and how to leave alone the pages +that do not need it. + +## When this matters, and when it doesn't + +It matters for pages that exist to answer a question: guides, comparisons, pricing explainers, +documentation, help articles, "what is X" pages. + +It mostly does not matter for: + +- **Homepages, product landing pages, contact and legal pages.** They are not answers. Do not + force question headings onto them to turn a check green. +- **Narrative or opinion writing.** An essay's value is the argument. Chopping it into Q&A + destroys it. Add a short summary at the top if you want something quotable. +- **Pages that are already well structured but get no citations.** Then the cause is probably + elsewhere: the page may not be fetchable (see `aeo-server-rendering`, `aeo-ai-crawler-access`), + or nothing off-site points to it. Do not rewrite a good page on a hunch. + +Be honest about what is known. Google's own guidance on AI features says there are no extra +requirements or special optimizations to appear in AI Overviews or AI Mode beyond normal SEO +fundamentals, and that no special schema is needed. There is no verified public recipe for how any +other engine picks a passage, so do not promise that a given rewrite will earn a citation. What you +can say is that clear, complete, well-structured text is easier to extract, and it is also better +for human readers. + +## Detect + +Rule IDs this skill owns, with what the audit actually checks (from `@advance-labs/scoring` and +`@advance-labs/html-parser`): + +| Rule | Passes when | Read it carefully | +|---|---|---| +| `aeo.question-headings` | At least one heading across **all audited pages** is question-like: it ends in `?`, or its first word is who, what, when, where, why, how, which, can, should, is, are, do, does or will | One heading anywhere is enough. "How to ..." and "Who we are" both count. A pass proves little. | +| `aeo.answerable-content` | Mean visible word count across pages is 300 or more **and** at least one `