diff --git a/bots/bots.go b/bots/bots.go index 3a68620..4dfdf94 100644 --- a/bots/bots.go +++ b/bots/bots.go @@ -43,13 +43,8 @@ var bots = map[string]bool{ "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview Analytics) Chrome/27.0.1453 Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7 AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-social)": true, "Mozilla/5.0/Chrome/12.0.742.112 (Macintosh; Intel Mac OS X 10_6_8)": true, - "Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/)": true, - "Mozilla/5.0 (compatible; AhrefsBot/5.1; +http://ahrefs.com/robot/)": true, "Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)": true, "Mozilla/5.0 (compatible; alexa site audit/1.0; +http://www.alexa.com/help/webmasters; )": true, - "Mozilla/5.0 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)": true, - "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)": true, - "Mozilla/5.0 (compatible; archive.org_bot; Wayback Machine Live Record; +http://archive.org/details/archive.org_bot)": true, "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)": true, "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)": true, "Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)": true, @@ -202,14 +197,11 @@ var bots = map[string]bool{ "Slackbot 1.0 (+https://api.slack.com/robots)": true, "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)": true, "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)": true, - "spider": true, - "spiderbot": true, - "Stratagems Kumo": true, - "TelegramBot": true, - "TelegramBot (like TwitterBot)": true, - "Traackr.com Bot": true, - "Twitterbot/1.0": true, - "voltron": true, + "spider": true, + "spiderbot": true, + "Stratagems Kumo": true, + "Traackr.com Bot": true, + "voltron": true, "Wotbox/2.01 (+http://www.wotbox.com/bot/)": true, "yacybot (freeworld/global; amd64 Linux 3.12.43-52.6-default; java 1.8.0_40; Europe/en) http://yacy.net/bot.html": true, "yacybot (/global; x86_64 Mac OS X 10.11.4; java 1.8.0_77; America/en) http://yacy.net/bot.html": true, @@ -220,6 +212,12 @@ var bots = map[string]bool{ var botPrefixes = []string{ "http.rb/", "PHP/", + "python-requests/", + "Mozilla/5.0 (compatible; AhrefsBot/", + "Twitterbot/", + "TelegramBot", + "Mozilla/5.0 (compatible; Applebot/", + "Mozilla/5.0 (compatible; archive.org_bot", } // IsBot returns whether or not the provided User-Agent string is a known bot diff --git a/bots/bots_test.go b/bots/bots_test.go index 82b15d1..5c6a2ab 100644 --- a/bots/bots_test.go +++ b/bots/bots_test.go @@ -4,11 +4,15 @@ import "testing" func TestIsBot(t *testing.T) { tests := map[string]bool{ - "Twitterbot/1.0": true, - "http.rb/2.2.2 (Mastodon/1.6.0; +https://insolente.im/)": true, - "http.rb/2.2.2 (Mastodon/1.5.1; +https://mastodon.cloud/)": true, - "http.rb/2.2.2 (Mastodon/1.6.0rc5; +https://mastodon.sdf.org/)": true, - "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; Touch; rv:11.0) like Gecko": false, + "Twitterbot/1.0": true, + "http.rb/2.2.2 (Mastodon/1.6.0; +https://insolente.im/)": true, + "http.rb/2.2.2 (Mastodon/1.5.1; +https://mastodon.cloud/)": true, + "http.rb/2.2.2 (Mastodon/1.6.0rc5; +https://mastodon.sdf.org/)": true, + "http.rb/3.2.0 (Mastodon/2.4.3; +https://qoto.org/)": true, + "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; Touch; rv:11.0) like Gecko": false, + "Mozilla/5.0 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)": true, + "Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)": true, + "Mozilla/5.0 (compatible; AhrefsBot/5.2; +http://ahrefs.com/robot/)": true, } for ua, r := range tests {