Merge branch 'master' of github.com:writeas/web-core

bots/bots.go

@@ -3,6 +3,8 @@
 // incrementing the view count.
 package bots
 
+import "strings"
+
 var bots = map[string]bool{
 	"ABACHOBot/8.14 (Windows NT 6.1 1.5; ko;)":                                 true,
 	"AddThis.com robot tech.support@clearspring.com":                           true,
@@ -215,11 +217,24 @@ var bots = map[string]bool{
 	"Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)": true,
 }
 
+var botPrefixes = []string{
+	"http.rb/2.2.2 (Mastodon",
+	"PHP/",
+}
+
 // IsBot returns whether or not the provided User-Agent string is a known bot
 // or crawler.
 func IsBot(ua string) bool {
+	if ua == "" {
+		return true
+	}
 	if _, ok := bots[ua]; ok {
 		return true
 	}
+	for _, p := range botPrefixes {
+		if strings.HasPrefix(ua, p) {
+			return true
+		}
+	}
 	return false
 }

bots/bots_test.go

@@ -0,0 +1,19 @@
+package bots
+
+import "testing"
+
+func TestIsBot(t *testing.T) {
+	tests := map[string]bool{
+		"Twitterbot/1.0":                                                               true,
+		"http.rb/2.2.2 (Mastodon/1.6.0; +https://insolente.im/)":                       true,
+		"http.rb/2.2.2 (Mastodon/1.5.1; +https://mastodon.cloud/)":                     true,
+		"http.rb/2.2.2 (Mastodon/1.6.0rc5; +https://mastodon.sdf.org/)":                true,
+		"Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; Touch; rv:11.0) like Gecko": false,
+	}
+
+	for ua, r := range tests {
+		if IsBot(ua) != r {
+			t.Errorf("Expected bot = %t on '%s'", r, ua)
+		}
+	}
+}