From 47655b3e663d4172126e757f752be775f00b70a1 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Sat, 23 Sep 2017 14:04:00 -0400 Subject: [PATCH 1/4] Mark Mastodon clients as bot --- bots/bots.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/bots/bots.go b/bots/bots.go index 809efc3..7b50986 100644 --- a/bots/bots.go +++ b/bots/bots.go @@ -3,6 +3,8 @@ // incrementing the view count. package bots +import "strings" + var bots = map[string]bool{ "ABACHOBot/8.14 (Windows NT 6.1 1.5; ko;)": true, "AddThis.com robot tech.support@clearspring.com": true, @@ -215,11 +217,20 @@ var bots = map[string]bool{ "Y!J-ASR/0.1 crawler (http://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716/)": true, } +var botPrefixes = []string{ + "http.rb/2.2.2 (Mastodon", +} + // IsBot returns whether or not the provided User-Agent string is a known bot // or crawler. func IsBot(ua string) bool { if _, ok := bots[ua]; ok { return true } + for _, p := range botPrefixes { + if strings.HasPrefix(ua, p) { + return true + } + } return false } From b80eea54540621cfb7017a23a3a8d6c0dacdf10c Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Sat, 23 Sep 2017 14:04:43 -0400 Subject: [PATCH 2/4] Add bots test --- bots/bots_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 bots/bots_test.go diff --git a/bots/bots_test.go b/bots/bots_test.go new file mode 100644 index 0000000..82b15d1 --- /dev/null +++ b/bots/bots_test.go @@ -0,0 +1,19 @@ +package bots + +import "testing" + +func TestIsBot(t *testing.T) { + tests := map[string]bool{ + "Twitterbot/1.0": true, + "http.rb/2.2.2 (Mastodon/1.6.0; +https://insolente.im/)": true, + "http.rb/2.2.2 (Mastodon/1.5.1; +https://mastodon.cloud/)": true, + "http.rb/2.2.2 (Mastodon/1.6.0rc5; +https://mastodon.sdf.org/)": true, + "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; Touch; rv:11.0) like Gecko": false, + } + + for ua, r := range tests { + if IsBot(ua) != r { + t.Errorf("Expected bot = %t on '%s'", r, ua) + } + } +} From e082f1e4026c34b8e485343af1a19046297d4aff Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 28 Sep 2017 17:19:51 -0400 Subject: [PATCH 3/4] Mark PHP clients as bots --- bots/bots.go | 1 + 1 file changed, 1 insertion(+) diff --git a/bots/bots.go b/bots/bots.go index 7b50986..ca21554 100644 --- a/bots/bots.go +++ b/bots/bots.go @@ -219,6 +219,7 @@ var bots = map[string]bool{ var botPrefixes = []string{ "http.rb/2.2.2 (Mastodon", + "PHP/", } // IsBot returns whether or not the provided User-Agent string is a known bot From 5c3827dc7695b1b21131ae2c92c404cd60444b89 Mon Sep 17 00:00:00 2001 From: Matt Baer Date: Thu, 28 Sep 2017 17:20:32 -0400 Subject: [PATCH 4/4] Mark empty User-Agent strings as bots --- bots/bots.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bots/bots.go b/bots/bots.go index ca21554..f0a008a 100644 --- a/bots/bots.go +++ b/bots/bots.go @@ -225,6 +225,9 @@ var botPrefixes = []string{ // IsBot returns whether or not the provided User-Agent string is a known bot // or crawler. func IsBot(ua string) bool { + if ua == "" { + return true + } if _, ok := bots[ua]; ok { return true }