Skip to content

Commit 3113869

Browse files
github-actions[bot]github-actions[bot]
committed
Update CrawlerScope data
1 parent 32815ec commit 3113869

3 files changed

Lines changed: 315 additions & 291 deletions

File tree

‎data/current/crawlers.json‎

Lines changed: 44 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@
193193
}
194194
]
195195
},
196-
"generatedAt": "2026-09-03T11:21:40Z",
196+
"generatedAt": "2026-09-03T16:29:56Z",
197197
"insights": [
198198
{
199199
"detail": "Total CIDR prefix change since the previous snapshot.",
@@ -227,7 +227,7 @@
227227
"documentationUrl": null,
228228
"id": "google-common",
229229
"ipListAuthoritative": true,
230-
"lastCheckedAt": "2026-09-03T11:21:40Z",
230+
"lastCheckedAt": "2026-09-03T16:29:56Z",
231231
"note": null,
232232
"operator": "Google",
233233
"operatorCountry": "US",
@@ -326,7 +326,7 @@
326326
"documentationUrl": null,
327327
"id": "google-special",
328328
"ipListAuthoritative": true,
329-
"lastCheckedAt": "2026-09-03T11:21:40Z",
329+
"lastCheckedAt": "2026-09-03T16:29:56Z",
330330
"note": null,
331331
"operator": "Google",
332332
"operatorCountry": "US",
@@ -400,7 +400,7 @@
400400
"documentationUrl": null,
401401
"id": "google-user-triggered",
402402
"ipListAuthoritative": true,
403-
"lastCheckedAt": "2026-09-03T11:21:40Z",
403+
"lastCheckedAt": "2026-09-03T16:29:56Z",
404404
"note": null,
405405
"operator": "Google",
406406
"operatorCountry": "US",
@@ -656,7 +656,7 @@
656656
"documentationUrl": null,
657657
"id": "bingbot",
658658
"ipListAuthoritative": true,
659-
"lastCheckedAt": "2026-09-03T11:21:40Z",
659+
"lastCheckedAt": "2026-09-03T16:29:56Z",
660660
"note": null,
661661
"operator": "Microsoft",
662662
"operatorCountry": "US",
@@ -720,7 +720,7 @@
720720
"documentationUrl": null,
721721
"id": "duckduckbot",
722722
"ipListAuthoritative": true,
723-
"lastCheckedAt": "2026-09-03T11:21:40Z",
723+
"lastCheckedAt": "2026-09-03T16:29:56Z",
724724
"note": null,
725725
"operator": "DuckDuckGo",
726726
"operatorCountry": "US",
@@ -1239,7 +1239,7 @@
12391239
"documentationUrl": null,
12401240
"id": "duckassistbot",
12411241
"ipListAuthoritative": true,
1242-
"lastCheckedAt": "2026-09-03T11:21:40Z",
1242+
"lastCheckedAt": "2026-09-03T16:29:56Z",
12431243
"note": null,
12441244
"operator": "DuckDuckGo",
12451245
"operatorCountry": "US",
@@ -1758,7 +1758,7 @@
17581758
"documentationUrl": "https://support.apple.com/en-us/119829",
17591759
"id": "applebot",
17601760
"ipListAuthoritative": true,
1761-
"lastCheckedAt": "2026-09-03T11:21:40Z",
1761+
"lastCheckedAt": "2026-09-03T16:29:56Z",
17621762
"note": null,
17631763
"operator": "Apple",
17641764
"operatorCountry": "US",
@@ -1814,7 +1814,7 @@
18141814
"documentationUrl": null,
18151815
"id": "mojeekbot",
18161816
"ipListAuthoritative": true,
1817-
"lastCheckedAt": "2026-09-03T11:21:40Z",
1817+
"lastCheckedAt": "2026-09-03T16:29:56Z",
18181818
"note": null,
18191819
"operator": "Mojeek",
18201820
"operatorCountry": "GB",
@@ -1850,7 +1850,7 @@
18501850
"documentationUrl": null,
18511851
"id": "naver-yeti",
18521852
"ipListAuthoritative": true,
1853-
"lastCheckedAt": "2026-09-03T11:21:40Z",
1853+
"lastCheckedAt": "2026-09-03T16:29:56Z",
18541854
"note": null,
18551855
"operator": "Naver",
18561856
"operatorCountry": "KR",
@@ -1921,7 +1921,7 @@
19211921
"documentationUrl": null,
19221922
"id": "yandexbot",
19231923
"ipListAuthoritative": false,
1924-
"lastCheckedAt": "2026-09-03T11:21:40Z",
1924+
"lastCheckedAt": "2026-09-03T16:29:56Z",
19251925
"note": "Static known ranges; verify live requests with reverse DNS and forward-confirmation.",
19261926
"operator": "Yandex",
19271927
"operatorCountry": "RU",
@@ -1970,7 +1970,7 @@
19701970
"documentationUrl": null,
19711971
"id": "baiduspider",
19721972
"ipListAuthoritative": false,
1973-
"lastCheckedAt": "2026-09-03T11:21:40Z",
1973+
"lastCheckedAt": "2026-09-03T16:29:56Z",
19741974
"note": "Small static seed list; not a complete authoritative Baidu crawler range list.",
19751975
"operator": "Baidu",
19761976
"operatorCountry": "CN",
@@ -2008,7 +2008,7 @@
20082008
"documentationUrl": null,
20092009
"id": "openai-gptbot",
20102010
"ipListAuthoritative": true,
2011-
"lastCheckedAt": "2026-09-03T11:21:40Z",
2011+
"lastCheckedAt": "2026-09-03T16:29:56Z",
20122012
"note": null,
20132013
"operator": "OpenAI",
20142014
"operatorCountry": "US",
@@ -2058,7 +2058,7 @@
20582058
"documentationUrl": null,
20592059
"id": "openai-searchbot",
20602060
"ipListAuthoritative": true,
2061-
"lastCheckedAt": "2026-09-03T11:21:40Z",
2061+
"lastCheckedAt": "2026-09-03T16:29:56Z",
20622062
"note": null,
20632063
"operator": "OpenAI",
20642064
"operatorCountry": "US",
@@ -2123,7 +2123,7 @@
21232123
"documentationUrl": null,
21242124
"id": "openai-chatgpt-user",
21252125
"ipListAuthoritative": true,
2126-
"lastCheckedAt": "2026-09-03T11:21:40Z",
2126+
"lastCheckedAt": "2026-09-03T16:29:56Z",
21272127
"note": null,
21282128
"operator": "OpenAI",
21292129
"operatorCountry": "US",
@@ -2349,7 +2349,7 @@
23492349
"documentationUrl": null,
23502350
"id": "openai-adsbot",
23512351
"ipListAuthoritative": false,
2352-
"lastCheckedAt": "2026-09-03T11:21:40Z",
2352+
"lastCheckedAt": "2026-09-03T16:29:56Z",
23532353
"note": "OpenAI documents this user-agent but does not publish a separate IP JSON list for it.",
23542354
"operator": "OpenAI",
23552355
"operatorCountry": "US",
@@ -2381,7 +2381,7 @@
23812381
"documentationUrl": null,
23822382
"id": "perplexitybot",
23832383
"ipListAuthoritative": true,
2384-
"lastCheckedAt": "2026-09-03T11:21:40Z",
2384+
"lastCheckedAt": "2026-09-03T16:29:56Z",
23852385
"note": null,
23862386
"operator": "Perplexity",
23872387
"operatorCountry": "US",
@@ -2422,7 +2422,7 @@
24222422
"documentationUrl": null,
24232423
"id": "perplexity-user",
24242424
"ipListAuthoritative": true,
2425-
"lastCheckedAt": "2026-09-03T11:21:40Z",
2425+
"lastCheckedAt": "2026-09-03T16:29:56Z",
24262426
"note": "Official ranges may not cover stealth or browser-like crawler behavior reported by third parties.",
24272427
"operator": "Perplexity",
24282428
"operatorCountry": "US",
@@ -2459,7 +2459,7 @@
24592459
"documentationUrl": null,
24602460
"id": "claudebot",
24612461
"ipListAuthoritative": false,
2462-
"lastCheckedAt": "2026-09-03T11:21:40Z",
2462+
"lastCheckedAt": "2026-09-03T16:29:56Z",
24632463
"note": "Anthropic documents crawler user-agents but says it does not publish stable IP ranges.",
24642464
"operator": "Anthropic",
24652465
"operatorCountry": "US",
@@ -2493,7 +2493,7 @@
24932493
"documentationUrl": "https://developer.amazon.com/support/amazonbot",
24942494
"id": "amazonbot",
24952495
"ipListAuthoritative": true,
2496-
"lastCheckedAt": "2026-09-03T11:21:40Z",
2496+
"lastCheckedAt": "2026-09-03T16:29:56Z",
24972497
"note": "Amazon publishes the list embedded in its developer page rather than as a standalone JSON endpoint.",
24982498
"operator": "Amazon",
24992499
"operatorCountry": "US",
@@ -3052,7 +3052,7 @@
30523052
"documentationUrl": "https://developer.amazon.com/support/amazonbot",
30533053
"id": "amazon-searchbot",
30543054
"ipListAuthoritative": true,
3055-
"lastCheckedAt": "2026-09-03T11:21:40Z",
3055+
"lastCheckedAt": "2026-09-03T16:29:56Z",
30563056
"note": "Amazon publishes the list embedded in its developer page rather than as a standalone JSON endpoint.",
30573057
"operator": "Amazon",
30583058
"operatorCountry": "US",
@@ -3599,7 +3599,7 @@
35993599
"documentationUrl": "https://developer.amazon.com/support/amazonbot",
36003600
"id": "amazon-amzn-user",
36013601
"ipListAuthoritative": true,
3602-
"lastCheckedAt": "2026-09-03T11:21:40Z",
3602+
"lastCheckedAt": "2026-09-03T16:29:56Z",
36033603
"note": "Amazon publishes the live crawl list embedded in its developer page rather than as a standalone JSON endpoint.",
36043604
"operator": "Amazon",
36053605
"operatorCountry": "US",
@@ -4657,7 +4657,7 @@
46574657
"documentationUrl": null,
46584658
"id": "meta-ai-crawlers",
46594659
"ipListAuthoritative": false,
4660-
"lastCheckedAt": "2026-09-03T11:21:40Z",
4660+
"lastCheckedAt": "2026-09-03T16:29:56Z",
46614661
"note": "Static Meta platform ranges, not crawler-specific authoritative ranges.",
46624662
"operator": "Meta",
46634663
"operatorCountry": "US",
@@ -4696,7 +4696,7 @@
46964696
"documentationUrl": null,
46974697
"id": "bytespider",
46984698
"ipListAuthoritative": false,
4699-
"lastCheckedAt": "2026-09-03T11:21:40Z",
4699+
"lastCheckedAt": "2026-09-03T16:29:56Z",
47004700
"note": null,
47014701
"operator": "ByteDance",
47024702
"operatorCountry": "CN",
@@ -4728,7 +4728,7 @@
47284728
"documentationUrl": null,
47294729
"id": "mistralai-user",
47304730
"ipListAuthoritative": true,
4731-
"lastCheckedAt": "2026-09-03T11:21:40Z",
4731+
"lastCheckedAt": "2026-09-03T16:29:56Z",
47324732
"note": null,
47334733
"operator": "Mistral AI",
47344734
"operatorCountry": "FR",
@@ -4765,7 +4765,7 @@
47654765
"documentationUrl": "https://help.ahrefs.com/articles/78658-what-is-the-list-of-your-ip-ranges",
47664766
"id": "ahrefsbot",
47674767
"ipListAuthoritative": true,
4768-
"lastCheckedAt": "2026-09-03T11:21:40Z",
4768+
"lastCheckedAt": "2026-09-03T16:29:56Z",
47694769
"note": null,
47704770
"operator": "Ahrefs",
47714771
"operatorCountry": "SG",
@@ -4882,7 +4882,7 @@
48824882
"documentationUrl": null,
48834883
"id": "lumar-crawler",
48844884
"ipListAuthoritative": true,
4885-
"lastCheckedAt": "2026-09-03T11:21:40Z",
4885+
"lastCheckedAt": "2026-09-03T16:29:56Z",
48864886
"note": null,
48874887
"operator": "Lumar",
48884888
"operatorCountry": "GB",
@@ -4981,7 +4981,7 @@
49814981
"documentationUrl": null,
49824982
"id": "semrushbot",
49834983
"ipListAuthoritative": false,
4984-
"lastCheckedAt": "2026-09-03T11:21:40Z",
4984+
"lastCheckedAt": "2026-09-03T16:29:56Z",
49854985
"note": null,
49864986
"operator": "Semrush",
49874987
"operatorCountry": "US",
@@ -5015,7 +5015,7 @@
50155015
"documentationUrl": null,
50165016
"id": "censys",
50175017
"ipListAuthoritative": false,
5018-
"lastCheckedAt": "2026-09-03T11:21:40Z",
5018+
"lastCheckedAt": "2026-09-03T16:29:56Z",
50195019
"note": null,
50205020
"operator": "Censys",
50215021
"operatorCountry": "US",
@@ -5052,7 +5052,7 @@
50525052
"documentationUrl": null,
50535053
"id": "shodan",
50545054
"ipListAuthoritative": false,
5055-
"lastCheckedAt": "2026-09-03T11:21:40Z",
5055+
"lastCheckedAt": "2026-09-03T16:29:56Z",
50565056
"note": "Community-known/static ranges; verify with reverse DNS and current Shodan documentation.",
50575057
"operator": "Shodan",
50585058
"operatorCountry": "US",
@@ -5097,7 +5097,7 @@
50975097
"documentationUrl": null,
50985098
"id": "datadog-synthetics",
50995099
"ipListAuthoritative": true,
5100-
"lastCheckedAt": "2026-09-03T11:21:40Z",
5100+
"lastCheckedAt": "2026-09-03T16:29:56Z",
51015101
"note": null,
51025102
"operator": "Datadog",
51035103
"operatorCountry": "US",
@@ -5239,7 +5239,7 @@
52395239
"documentationUrl": null,
52405240
"id": "ias-crawler",
52415241
"ipListAuthoritative": true,
5242-
"lastCheckedAt": "2026-09-03T11:21:40Z",
5242+
"lastCheckedAt": "2026-09-03T16:29:56Z",
52435243
"note": null,
52445244
"operator": "Integral Ad Science",
52455245
"operatorCountry": "US",
@@ -5286,7 +5286,7 @@
52865286
"documentationUrl": null,
52875287
"id": "ttd-content",
52885288
"ipListAuthoritative": true,
5289-
"lastCheckedAt": "2026-09-03T11:21:40Z",
5289+
"lastCheckedAt": "2026-09-03T16:29:56Z",
52905290
"note": null,
52915291
"operator": "The Trade Desk",
52925292
"operatorCountry": "US",
@@ -7934,7 +7934,7 @@
79347934
"documentationUrl": "https://uptimerobot.com/locations",
79357935
"id": "uptimerobot",
79367936
"ipListAuthoritative": true,
7937-
"lastCheckedAt": "2026-09-03T11:21:40Z",
7937+
"lastCheckedAt": "2026-09-03T16:29:56Z",
79387938
"note": null,
79397939
"operator": "UptimeRobot",
79407940
"operatorCountry": "US",
@@ -8173,7 +8173,7 @@
81738173
"documentationUrl": null,
81748174
"id": "pingdom",
81758175
"ipListAuthoritative": true,
8176-
"lastCheckedAt": "2026-09-03T11:21:40Z",
8176+
"lastCheckedAt": "2026-09-03T16:29:56Z",
81778177
"note": null,
81788178
"operator": "SolarWinds Pingdom",
81798179
"operatorCountry": "US",
@@ -8412,7 +8412,7 @@
84128412
"documentationUrl": "https://www.statuscake.com/kb/knowledge-base/what-are-your-ips/",
84138413
"id": "statuscake",
84148414
"ipListAuthoritative": true,
8415-
"lastCheckedAt": "2026-09-03T11:21:40Z",
8415+
"lastCheckedAt": "2026-09-03T16:29:56Z",
84168416
"note": null,
84178417
"operator": "StatusCake",
84188418
"operatorCountry": "GB",
@@ -8742,7 +8742,7 @@
87428742
"documentationUrl": "https://betterstack.com/docs/uptime/frequently-asked-questions/",
87438743
"id": "better-stack",
87448744
"ipListAuthoritative": true,
8745-
"lastCheckedAt": "2026-09-03T11:21:40Z",
8745+
"lastCheckedAt": "2026-09-03T16:29:56Z",
87468746
"note": "Better Stack publishes probe addresses through its Better Uptime IP text endpoint.",
87478747
"operator": "Better Stack",
87488748
"operatorCountry": "CZ",
@@ -8810,7 +8810,7 @@
88108810
"documentationUrl": null,
88118811
"id": "ccbot",
88128812
"ipListAuthoritative": true,
8813-
"lastCheckedAt": "2026-09-03T11:21:40Z",
8813+
"lastCheckedAt": "2026-09-03T16:29:56Z",
88148814
"note": null,
88158815
"operator": "Common Crawl",
88168816
"operatorCountry": "US",
@@ -8849,7 +8849,7 @@
88498849
"documentationUrl": null,
88508850
"id": "flipboard-crawler",
88518851
"ipListAuthoritative": true,
8852-
"lastCheckedAt": "2026-09-03T11:21:40Z",
8852+
"lastCheckedAt": "2026-09-03T16:29:56Z",
88538853
"note": null,
88548854
"operator": "Flipboard",
88558855
"operatorCountry": "US",
@@ -9018,7 +9018,7 @@
90189018
"documentationUrl": null,
90199019
"id": "parsely-crawler",
90209020
"ipListAuthoritative": true,
9021-
"lastCheckedAt": "2026-09-03T11:21:40Z",
9021+
"lastCheckedAt": "2026-09-03T16:29:56Z",
90229022
"note": null,
90239023
"operator": "Parse.ly",
90249024
"operatorCountry": "US",
@@ -9061,7 +9061,7 @@
90619061
"documentationUrl": null,
90629062
"id": "pinterestbot",
90639063
"ipListAuthoritative": false,
9064-
"lastCheckedAt": "2026-09-03T11:21:40Z",
9064+
"lastCheckedAt": "2026-09-03T16:29:56Z",
90659065
"note": null,
90669066
"operator": "Pinterest",
90679067
"operatorCountry": "US",
@@ -9095,7 +9095,7 @@
90959095
"documentationUrl": null,
90969096
"id": "linkedinbot",
90979097
"ipListAuthoritative": false,
9098-
"lastCheckedAt": "2026-09-03T11:21:40Z",
9098+
"lastCheckedAt": "2026-09-03T16:29:56Z",
90999099
"note": null,
91009100
"operator": "LinkedIn",
91019101
"operatorCountry": "US",
@@ -9127,7 +9127,7 @@
91279127
"documentationUrl": "https://core.telegram.org/bots/webapps#link-preview",
91289128
"id": "telegram-link-preview",
91299129
"ipListAuthoritative": true,
9130-
"lastCheckedAt": "2026-09-03T11:21:40Z",
9130+
"lastCheckedAt": "2026-09-03T16:29:56Z",
91319131
"note": null,
91329132
"operator": "Telegram",
91339133
"operatorCountry": "AE",
@@ -9172,7 +9172,7 @@
91729172
"documentationUrl": "https://rssapi.net/faq",
91739173
"id": "rssapi-parser",
91749174
"ipListAuthoritative": true,
9175-
"lastCheckedAt": "2026-09-03T11:21:40Z",
9175+
"lastCheckedAt": "2026-09-03T16:29:56Z",
91769176
"note": null,
91779177
"operator": "RSS API",
91789178
"operatorCountry": "DE",

0 commit comments

Comments
 (0)