cyber-mawonaj/web-core-fr-gp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move explicit bot UA string tests to prefix tests

Browse Source 
Also look for python-requests/*
This commit is contained in:
Matt Baer
2018-08-04 17:28:29 +02:00
parent 0f417dd897
commit 582ab04ab5
2 changed files with 20 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bots/bots.go
+6 -8
View File
@@ -43,13 +43,8 @@ var bots = map[string]bool{
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview Analytics) Chrome/27.0.1453 Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true, "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview Analytics) Chrome/27.0.1453 Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true,
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7 AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-social)": true, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7 AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-social)": true,
"Mozilla/5.0/Chrome/12.0.742.112 (Macintosh; Intel Mac OS X 10_6_8)": true, "Mozilla/5.0/Chrome/12.0.742.112 (Macintosh; Intel Mac OS X 10_6_8)": true,
"Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/)": true,
"Mozilla/5.0 (compatible; AhrefsBot/5.1; +http://ahrefs.com/robot/)": true,
"Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)": true, "Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)": true,
"Mozilla/5.0 (compatible; alexa site audit/1.0; +http://www.alexa.com/help/webmasters; )": true, "Mozilla/5.0 (compatible; alexa site audit/1.0; +http://www.alexa.com/help/webmasters; )": true,
"Mozilla/5.0 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)": true,
"Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)": true,
"Mozilla/5.0 (compatible; archive.org_bot; Wayback Machine Live Record; +http://archive.org/details/archive.org_bot)": true,
"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)": true, "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)": true,
"Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)": true, "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)": true,
"Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)": true, "Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)": true,
@@ -205,10 +200,7 @@ var bots = map[string]bool{
"spider": true, "spider": true,
"spiderbot": true, "spiderbot": true,
"Stratagems Kumo": true, "Stratagems Kumo": true,
"TelegramBot": true,
"TelegramBot (like TwitterBot)": true,
"Traackr.com Bot": true, "Traackr.com Bot": true,
"Twitterbot/1.0": true,
"voltron": true, "voltron": true,
"Wotbox/2.01 (+http://www.wotbox.com/bot/)": true, "Wotbox/2.01 (+http://www.wotbox.com/bot/)": true,
"yacybot (freeworld/global; amd64 Linux 3.12.43-52.6-default; java 1.8.0_40; Europe/en) http://yacy.net/bot.html": true, "yacybot (freeworld/global; amd64 Linux 3.12.43-52.6-default; java 1.8.0_40; Europe/en) http://yacy.net/bot.html": true,
@@ -220,6 +212,12 @@ var bots = map[string]bool{
var botPrefixes = []string{ var botPrefixes = []string{
"http.rb/", "http.rb/",
"PHP/", "PHP/",
"python-requests/",
"Mozilla/5.0 (compatible; AhrefsBot/",
"Twitterbot/",
"TelegramBot",
"Mozilla/5.0 (compatible; Applebot/",
"Mozilla/5.0 (compatible; archive.org_bot",
} }
// IsBot returns whether or not the provided User-Agent string is a known bot // IsBot returns whether or not the provided User-Agent string is a known bot
bots/bots_test.go
+4
View File
@@ -8,7 +8,11 @@ func TestIsBot(t *testing.T) {
"http.rb/2.2.2 (Mastodon/1.6.0; +https://insolente.im/)": true, "http.rb/2.2.2 (Mastodon/1.6.0; +https://insolente.im/)": true,
"http.rb/2.2.2 (Mastodon/1.5.1; +https://mastodon.cloud/)": true, "http.rb/2.2.2 (Mastodon/1.5.1; +https://mastodon.cloud/)": true,
"http.rb/2.2.2 (Mastodon/1.6.0rc5; +https://mastodon.sdf.org/)": true, "http.rb/2.2.2 (Mastodon/1.6.0rc5; +https://mastodon.sdf.org/)": true,
"http.rb/3.2.0 (Mastodon/2.4.3; +https://qoto.org/)": true,
"Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; Touch; rv:11.0) like Gecko": false, "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; Touch; rv:11.0) like Gecko": false,
"Mozilla/5.0 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)": true,
"Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)": true,
"Mozilla/5.0 (compatible; AhrefsBot/5.2; +http://ahrefs.com/robot/)": true,
} }
for ua, r := range tests { for ua, r := range tests {