Skip to content

Commit f23f690

Browse files
committed
Added OpenAI and Claude bots.
Added tests. Updated dependencies.
1 parent b3892bd commit f23f690

File tree

3 files changed

+61
-29
lines changed

3 files changed

+61
-29
lines changed

composer.lock

Lines changed: 27 additions & 26 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

src/mappings/crawlers.php

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,8 @@ public static function getApp(string $value, array $data = []) : array {
6565
'yisouspider' => 'search',
6666
'360spider' => 'search',
6767
'sogou web spider' => 'search',
68-
'bytespider' => 'crawler'
68+
'bytespider' => 'crawler',
69+
'claudebot' => 'ai'
6970
];
7071
$apps = [
7172
'yacybot' => 'YacyBot',
@@ -124,7 +125,9 @@ public static function getApp(string $value, array $data = []) : array {
124125
'googledocs' => 'Google Docs',
125126
'user-agent: seolyt' => 'SEOlyt',
126127
'bytespider' => 'ByteDance Spider',
127-
'spider-feedback@bytedance.com' => 'ByteDance Spider'
128+
'spider-feedback@bytedance.com' => 'ByteDance Spider',
129+
'oai-searchbot' => 'OpenAI SearchBot',
130+
'chatgpt-user' => 'ChatGPT User'
128131
];
129132

130133
$lower = \mb_strtolower($parts[0]);
@@ -391,14 +394,15 @@ public static function get() : array {
391394
'http-client/' => new props('any', $fn['scraper']),
392395
'HttpClient/' => new props('any', $fn['scraper']),
393396
'PowerShell/' => new props('start', $fn['scraper']),
397+
'OAI-SearchBot/' => new props('start', $fn['search']),
394398
'GPTBot/' => new props('start', $fn['ai']),
395399
'Diffbot/' => new props('start', $fn['ai']),
396400
'Amazonbot/' => new props('start', $fn['ai']),
397401
'Applebot/' => new props('start', $fn['ai']),
398402
'PerplexityBot/' => new props('start', $fn['ai']),
399403
'YouBot/' => new props('start', $fn['ai']),
400404
'Google-Extended' => new props('start', $fn['ai']),
401-
'ChatGPT-User/' => new props('start', $fn['feed']),
405+
'ChatGPT-User/' => new props('start', $fn['ai']),
402406
'facebookexternalhit/' => new props('start', $fn['feed']),
403407
'facebookcatalog/' => new props('start', $fn['feed']),
404408
'Validator' => new props('any', $fn['validator']),

tests/crawlersTest.php

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -411,6 +411,15 @@ public function testSearch() : void {
411411
'appname' => 'YandexRenderResourcesBot',
412412
'appversion' => '1.0',
413413
'url' => 'http://yandex.com/bots'
414+
],
415+
'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot' => [
416+
'string' => 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot',
417+
'type' => 'robot',
418+
'category' => 'search',
419+
'app' => 'OpenAI SearchBot',
420+
'appname' => 'OAI-SearchBot',
421+
'appversion' => '1.0',
422+
'url' => 'https://openai.com/searchbot'
414423
]
415424
];
416425
foreach ($strings AS $ua => $item) {
@@ -589,6 +598,15 @@ public function testFeed() : void {
589598
'appname' => 'filemanager.downloads.http.client - contentfilemanagerdaemons-downloadFromUrlTq2Worke',
590599
'appversion' => '2.0',
591600
'url' => 'http://dev.hubspot.com/'
601+
],
602+
'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot' => [
603+
'string' => 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot',
604+
'type' => 'robot',
605+
'category' => 'ai',
606+
'app' => 'ChatGPT User',
607+
'appname' => 'ChatGPT-User',
608+
'appversion' => '1.0',
609+
'url' => 'https://openai.com/bot'
592610
]
593611
];
594612
foreach ($strings AS $ua => $item) {
@@ -1278,6 +1296,15 @@ public function testAi() : void {
12781296
'appname' => 'YouBot',
12791297
'appversion' => '1.0',
12801298
'url' => 'https://about.you.com/youbot/'
1299+
],
1300+
'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.1; +https://openai.com/gptbot' => [
1301+
'string' => 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.1; +https://openai.com/gptbot',
1302+
'type' => 'robot',
1303+
'category' => 'ai',
1304+
'app' => 'GPTBot',
1305+
'appname' => 'GPTBot',
1306+
'appversion' => '1.1',
1307+
'url' => 'https://openai.com/gptbot'
12811308
]
12821309
];
12831310
foreach ($strings AS $ua => $item) {

0 commit comments

Comments
 (0)