diff --git a/bots/bots.go b/bots/bots.go index 809efc3..f0a008a 100644 --- a/bots/bots.go +++ b/bots/bots.go @@ -3,6 +3,8 @@ // incrementing the view count. package bots +import "strings" + var bots = map[string]bool{ "ABACHOBot/8.14 (Windows NT 6.1 1.5; ko;)": true, "AddThis.com robot tech.support@clearspring.com": true, @@ -215,11 +217,24 @@ var bots = map[string]bool{ "Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)": true, } +var botPrefixes = []string{ + "http.rb/2.2.2 (Mastodon", + "PHP/", +} + // IsBot returns whether or not the provided User-Agent string is a known bot // or crawler. func IsBot(ua string) bool { + if ua == "" { + return true + } if _, ok := bots[ua]; ok { return true } + for _, p := range botPrefixes { + if strings.HasPrefix(ua, p) { + return true + } + } return false } diff --git a/bots/bots_test.go b/bots/bots_test.go new file mode 100644 index 0000000..82b15d1 --- /dev/null +++ b/bots/bots_test.go @@ -0,0 +1,19 @@ +package bots + +import "testing" + +func TestIsBot(t *testing.T) { + tests := map[string]bool{ + "Twitterbot/1.0": true, + "http.rb/2.2.2 (Mastodon/1.6.0; +https://insolente.im/)": true, + "http.rb/2.2.2 (Mastodon/1.5.1; +https://mastodon.cloud/)": true, + "http.rb/2.2.2 (Mastodon/1.6.0rc5; +https://mastodon.sdf.org/)": true, + "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; Touch; rv:11.0) like Gecko": false, + } + + for ua, r := range tests { + if IsBot(ua) != r { + t.Errorf("Expected bot = %t on '%s'", r, ua) + } + } +}