OffCryptPage/Robots.txt at main · EmptyCode0x86/OffCryptPage · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
# OffCrypt - Security & Encryption Tools
# https://www.dev-offcode.com

# Allow all web crawlers to access all content
# (Includes AI retrieval crawlers such as GPTBot, ClaudeBot, PerplexityBot via User-agent: *)
User-agent: *
Allow: /
Disallow: /private/
Disallow: /admin/
Disallow: /temp/
Disallow: /*.log$
Disallow: /*.tmp$

# Specific crawler instructions
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 2

User-agent: facebookexternalhit
Allow: /

# Block bad bots
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# Sitemaps
Sitemap: https://www.dev-offcode.com/sitemap.xml
Sitemap: https://www.dev-offcode.com/sitemap-images.xml

# Host declaration
Host: www.dev-offcode.com