Re-gofmt bots.go
This commit is contained in:
Matt Baer
+97
-97
@@ -6,113 +6,113 @@ package bots
|
|||||||
import "strings"
|
import "strings"
|
||||||
|
|
||||||
var bots = map[string]bool{
|
var bots = map[string]bool{
|
||||||
"ABACHOBot/8.14 (Windows NT 6.1 1.5; ko;)": true,
|
"ABACHOBot/8.14 (Windows NT 6.1 1.5; ko;)": true,
|
||||||
"AddThis.com robot tech.support@clearspring.com": true,
|
"AddThis.com robot tech.support@clearspring.com": true,
|
||||||
"Alwyzbot/1.0": true,
|
"Alwyzbot/1.0": true,
|
||||||
"Apache-HttpClient/4.2.2 (java 1.5)": true,
|
"Apache-HttpClient/4.2.2 (java 1.5)": true,
|
||||||
"bitlybot": true,
|
"bitlybot": true,
|
||||||
"bitlybot/3.0 (+http://bit.ly/)": true,
|
"bitlybot/3.0 (+http://bit.ly/)": true,
|
||||||
"Buzzbot/1.0 (Buzzbot; http://www.buzzstream.com; buzzbot@buzzstream.com)": true,
|
"Buzzbot/1.0 (Buzzbot; http://www.buzzstream.com; buzzbot@buzzstream.com)": true,
|
||||||
"cis455crawler": true,
|
"cis455crawler": true,
|
||||||
"COMODO SSL Checker": true,
|
"COMODO SSL Checker": true,
|
||||||
"crawlernutchtest/Nutch-1.9": true,
|
"crawlernutchtest/Nutch-1.9": true,
|
||||||
"CRAZYWEBCRAWLER 0.9.8, http://www.crazywebcrawler.com": true,
|
"CRAZYWEBCRAWLER 0.9.8, http://www.crazywebcrawler.com": true,
|
||||||
"CSS Certificate Spider (http://www.css-security.com/certificatespider/)": true,
|
"CSS Certificate Spider (http://www.css-security.com/certificatespider/)": true,
|
||||||
"datebot": true,
|
"datebot": true,
|
||||||
"DeadYetBot/1.0 (+http://deadyet.lol)": true,
|
"DeadYetBot/1.0 (+http://deadyet.lol)": true,
|
||||||
"dj-research/Nutch-1.11 (analytics@@demandjump.com)": true,
|
"dj-research/Nutch-1.11 (analytics@@demandjump.com)": true,
|
||||||
"DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)": true,
|
"DoCoMo/2.0 N905i(c100;TB;W24H16) (compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)": true,
|
||||||
"DomainCrawler/1.0": true,
|
"DomainCrawler/1.0": true,
|
||||||
"Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com": true,
|
"Domain Re-Animator Bot (http://domainreanimator.com) - support@domainreanimator.com": true,
|
||||||
"ExactSeekCrawler/1.0": true,
|
"ExactSeekCrawler/1.0": true,
|
||||||
"Gigabot/1.0": true,
|
"Gigabot/1.0": true,
|
||||||
"Go 1.1 package http": true,
|
"Go 1.1 package http": true,
|
||||||
"Googlebot/2.1 (+http://www.googlebot.com/bot.html)": true,
|
"Googlebot/2.1 (+http://www.googlebot.com/bot.html)": true,
|
||||||
"Googlebot/2.1 (+http://www.google.com/bot.html)": true,
|
"Googlebot/2.1 (+http://www.google.com/bot.html)": true,
|
||||||
"Googlebot-Image/1.0": true,
|
"Googlebot-Image/1.0": true,
|
||||||
"libwww-perl/6.04": true,
|
"libwww-perl/6.04": true,
|
||||||
"LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)": true,
|
"LinkedInBot/1.0 (compatible; Mozilla/5.0; Jakarta Commons-HttpClient/3.1 +http://www.linkedin.com)": true,
|
||||||
"LivelapBot/0.2 (http://site.livelap.com/crawler)": true,
|
"LivelapBot/0.2 (http://site.livelap.com/crawler)": true,
|
||||||
"LSSRocketCrawler/1.0 LightspeedSystems": true,
|
"LSSRocketCrawler/1.0 LightspeedSystems": true,
|
||||||
"Mediatoolkitbot (complaints@mediatoolkit.com)": true,
|
"Mediatoolkitbot (complaints@mediatoolkit.com)": true,
|
||||||
"Mediatoolkitbot (info@mediatoolkit.com)": true,
|
"Mediatoolkitbot (info@mediatoolkit.com)": true,
|
||||||
"Mediumbot-MetaTagFetcher/0.1 (+https://medium.com/)": true,
|
"Mediumbot-MetaTagFetcher/0.1 (+https://medium.com/)": true,
|
||||||
"Melvil/1.0": true,
|
"Melvil/1.0": true,
|
||||||
"MetaCommentBot; http://metacomment.io/about": true,
|
"MetaCommentBot; http://metacomment.io/about": true,
|
||||||
"mfibot/1.1 (http://www.mfisoft.ru/analyst/; <admin@mfisoft.ru>; en-RU)": true,
|
"mfibot/1.1 (http://www.mfisoft.ru/analyst/; <admin@mfisoft.ru>; en-RU)": true,
|
||||||
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview Analytics) Chrome/27.0.1453 Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true,
|
"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; Google Web Preview Analytics) Chrome/27.0.1453 Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true,
|
||||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7 AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-social)": true,
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7 AppEngine-Google; (+http://code.google.com/appengine; appid: s~feedly-social)": true,
|
||||||
"Mozilla/5.0/Chrome/12.0.742.112 (Macintosh; Intel Mac OS X 10_6_8)": true,
|
"Mozilla/5.0/Chrome/12.0.742.112 (Macintosh; Intel Mac OS X 10_6_8)": true,
|
||||||
"Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)": true,
|
"Mozilla/5.0 (compatible; aiHitBot/2.9; +https://www.aihitdata.com/about)": true,
|
||||||
"Mozilla/5.0 (compatible; alexa site audit/1.0; +http://www.alexa.com/help/webmasters; )": true,
|
"Mozilla/5.0 (compatible; alexa site audit/1.0; +http://www.alexa.com/help/webmasters; )": true,
|
||||||
"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)": true,
|
"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)": true,
|
||||||
"Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)": true,
|
"Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)": true,
|
||||||
"Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)": true,
|
"Mozilla/5.0 (compatible; BLEXBot/1.0; +http://webmeup-crawler.com/)": true,
|
||||||
"Mozilla/5.0 (compatible; BuzzSumo; +http://www.buzzsumo.com/bot.html)": true,
|
"Mozilla/5.0 (compatible; BuzzSumo; +http://www.buzzsumo.com/bot.html)": true,
|
||||||
"Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)": true,
|
"Mozilla/5.0 (compatible; coccoc/1.0; +http://help.coccoc.com/)": true,
|
||||||
"Mozilla/5.0 (compatible; Discordbot/1.0; +https://discordapp.com)": true,
|
"Mozilla/5.0 (compatible; Discordbot/1.0; +https://discordapp.com)": true,
|
||||||
"Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)": true,
|
"Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)": true,
|
||||||
"Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)": true,
|
"Mozilla/5.0 (compatible; DuckDuckGo-Favicons-Bot/1.0; +http://duckduckgo.com)": true,
|
||||||
"Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)": true,
|
"Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)": true,
|
||||||
"Mozilla/5.0 (compatible; Feedspotbot/1.0; +http://www.feedspot.com/fs/bot)": true,
|
"Mozilla/5.0 (compatible; Feedspotbot/1.0; +http://www.feedspot.com/fs/bot)": true,
|
||||||
"Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)": true,
|
"Mozilla/5.0 (compatible; Findxbot/1.0; +http://www.findxbot.com)": true,
|
||||||
"Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)": true,
|
"Mozilla/5.0 (compatible; Gluten Free Crawler/1.0; +http://glutenfreepleasure.com/)": true,
|
||||||
"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true,
|
"Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true,
|
||||||
"Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20140702-2247 +http://archive.org/details/archive.org_bot)": true,
|
"Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-20140702-2247 +http://archive.org/details/archive.org_bot)": true,
|
||||||
"Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-2014-11-14T15:29:34Z +http://citeseerx.ist.psu.edu/)": true,
|
"Mozilla/5.0 (compatible; heritrix/3.3.0-SNAPSHOT-2014-11-14T15:29:34Z +http://citeseerx.ist.psu.edu/)": true,
|
||||||
"Mozilla/5.0/(compatible; heritrix/3.3.0-SNAPSHOT-20150803-2130 +http://literatur-im-netz.dla-marbach.de)": true,
|
"Mozilla/5.0/(compatible; heritrix/3.3.0-SNAPSHOT-20150803-2130 +http://literatur-im-netz.dla-marbach.de)": true,
|
||||||
"Mozilla/5.0 (compatible; Kraken/0.1; http://linkfluence.net/; bot@linkfluence.net)": true,
|
"Mozilla/5.0 (compatible; Kraken/0.1; http://linkfluence.net/; bot@linkfluence.net)": true,
|
||||||
"Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/)": true,
|
"Mozilla/5.0 (compatible; linkdexbot/2.2; +http://www.linkdex.com/bots/)": true,
|
||||||
"Mozilla/5.0 (compatible; LinkFeatureBot)": true,
|
"Mozilla/5.0 (compatible; LinkFeatureBot)": true,
|
||||||
"Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com)": true,
|
"Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com)": true,
|
||||||
"Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com) (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) Mobile/12H321": true,
|
"Mozilla/5.0 (compatible; LinkisBot/1.0; bot@linkis.com) (iPhone; CPU iPhone OS 8_4_1 like Mac OS X) Mobile/12H321": true,
|
||||||
"Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)": true,
|
"Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/2.0; +http://go.mail.ru/help/robots)": true,
|
||||||
"Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Robots/2.0; +http://go.mail.ru/help/robots)": true,
|
"Mozilla/5.0 (compatible; Linux x86_64; Mail.RU_Bot/Robots/2.0; +http://go.mail.ru/help/robots)": true,
|
||||||
"Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)": true,
|
"Mozilla/5.0 (compatible; Lipperhey-Kaus-Australis/5.0; +https://www.lipperhey.com/en/about/)": true,
|
||||||
"Mozilla/5.0 (compatible; meanpathbot/1.0; +http://www.meanpath.com/meanpathbot.html)": true,
|
"Mozilla/5.0 (compatible; meanpathbot/1.0; +http://www.meanpath.com/meanpathbot.html)": true,
|
||||||
"Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)": true,
|
"Mozilla/5.0 (compatible; MegaIndex.ru/2.0; +http://megaindex.com/crawler)": true,
|
||||||
"Mozilla/5.0 (compatible; MixrankBot; crawler@mixrank.com)": true,
|
"Mozilla/5.0 (compatible; MixrankBot; crawler@mixrank.com)": true,
|
||||||
"Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)": true,
|
"Mozilla/5.0 (compatible; MJ12bot/v1.4.5; http://www.majestic12.co.uk/bot.php?+)": true,
|
||||||
"Mozilla/5.0 (compatible; MojeekBot/0.6; +https://www.mojeek.com/bot.html)": true,
|
"Mozilla/5.0 (compatible; MojeekBot/0.6; +https://www.mojeek.com/bot.html)": true,
|
||||||
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)": true,
|
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)": true,
|
||||||
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; abot v1.2.3.1 http://code.google.com/p/abot)": true,
|
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; abot v1.2.3.1 http://code.google.com/p/abot)": true,
|
||||||
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider": true,
|
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider": true,
|
||||||
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)": true,
|
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0); 360Spider(compatible; HaosouSpider; http://www.haosou.com/help/help_3_2.html)": true,
|
||||||
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)": true,
|
"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)": true,
|
||||||
"Mozilla/5.0 (compatible; MSIE or Firefox mutant;) Daum 4.1": true,
|
"Mozilla/5.0 (compatible; MSIE or Firefox mutant;) Daum 4.1": true,
|
||||||
"Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daum 4.1": true,
|
"Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daum 4.1": true,
|
||||||
"Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa 4.0": true,
|
"Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa 4.0": true,
|
||||||
"Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)": true,
|
"Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)": true,
|
||||||
"Mozilla/5.0 (compatible; oBot/2.3.1; +http://filterdb.iss.net/crawler/)": true,
|
"Mozilla/5.0 (compatible; oBot/2.3.1; +http://filterdb.iss.net/crawler/)": true,
|
||||||
"Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)": true,
|
"Mozilla/5.0 (compatible; OpenHoseBot/2.1; +http://www.openhose.org/bot.html)": true,
|
||||||
"Mozilla/5.0 (compatible; OpsBot/1.0)": true,
|
"Mozilla/5.0 (compatible; OpsBot/1.0)": true,
|
||||||
"Mozilla/5.0 (compatible; OptimizationCrawler/0.2; +http://www.domainoptima.com/robot)": true,
|
"Mozilla/5.0 (compatible; OptimizationCrawler/0.2; +http://www.domainoptima.com/robot)": true,
|
||||||
"Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)": true,
|
"Mozilla/5.0 (compatible; PaperLiBot/2.1; http://support.paper.li/entries/20023257-what-is-paper-li)": true,
|
||||||
"Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)": true,
|
"Mozilla/5.0 (compatible; PrivacyAwareBot/1.1; +http://www.privacyaware.org)": true,
|
||||||
"Mozilla/5.0 (compatible; Qwantify/2.2w; +https://www.qwant.com/)/*": true,
|
"Mozilla/5.0 (compatible; Qwantify/2.2w; +https://www.qwant.com/)/*": true,
|
||||||
"Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)": true,
|
"Mozilla/5.0 (compatible; redditbot/1.0; +http://www.reddit.com/feedback)": true,
|
||||||
"Mozilla/5.0 (compatible; Sadakura; +http://shorf.com/bot.php)": true,
|
"Mozilla/5.0 (compatible; Sadakura; +http://shorf.com/bot.php)": true,
|
||||||
"Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)": true,
|
"Mozilla/5.0 (compatible; SemrushBot/1.1~bl; +http://www.semrush.com/bot.html)": true,
|
||||||
"Mozilla/5.0 (compatible; SemrushBot/1~bl; +http://www.semrush.com/bot.html)": true,
|
"Mozilla/5.0 (compatible; SemrushBot/1~bl; +http://www.semrush.com/bot.html)": true,
|
||||||
"Mozilla/5.0 (compatible; SEOkicks-Robot; +http://www.seokicks.de/robot.html)": true,
|
"Mozilla/5.0 (compatible; SEOkicks-Robot; +http://www.seokicks.de/robot.html)": true,
|
||||||
"Mozilla/5.0 (compatible; SEOlyticsCrawler/3.0; +http://crawler.seolytics.net/)": true,
|
"Mozilla/5.0 (compatible; SEOlyticsCrawler/3.0; +http://crawler.seolytics.net/)": true,
|
||||||
"Mozilla/5.0 (compatible; SeznamBot/3.2; +http://fulltext.sblog.cz/)": true,
|
"Mozilla/5.0 (compatible; SeznamBot/3.2; +http://fulltext.sblog.cz/)": true,
|
||||||
"Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)": true,
|
"Mozilla/5.0 (compatible; SeznamBot/3.2; +http://napoveda.seznam.cz/en/seznambot-intro/)": true,
|
||||||
"Mozilla/5.0 (compatible; spbot/4.4.2; +http://OpenLinkProfiler.org/bot )": true,
|
"Mozilla/5.0 (compatible; spbot/4.4.2; +http://OpenLinkProfiler.org/bot )": true,
|
||||||
"Mozilla/5.0 (compatible; spbot/5.0.1; +http://OpenLinkProfiler.org/bot )": true,
|
"Mozilla/5.0 (compatible; spbot/5.0.1; +http://OpenLinkProfiler.org/bot )": true,
|
||||||
"Mozilla/5.0 (compatible; spbot/5.0.2; +http://OpenLinkProfiler.org/bot )": true,
|
"Mozilla/5.0 (compatible; spbot/5.0.2; +http://OpenLinkProfiler.org/bot )": true,
|
||||||
"Mozilla/5.0 (compatible; spbot/5.0; +http://OpenLinkProfiler.org/bot )": true,
|
"Mozilla/5.0 (compatible; spbot/5.0; +http://OpenLinkProfiler.org/bot )": true,
|
||||||
"Mozilla/5.0 (compatible; special_archiver/3.1.1 +http://www.archive.org/details/archive.org_bot)": true,
|
"Mozilla/5.0 (compatible; special_archiver/3.1.1 +http://www.archive.org/details/archive.org_bot)": true,
|
||||||
"Mozilla/5.0 (compatible; TestCrawler)": true,
|
"Mozilla/5.0 (compatible; TestCrawler)": true,
|
||||||
"Mozilla/5.0 (compatible; uMBot-LN/1.0; mailto: crawling@ubermetrics-technologies.com)": true,
|
"Mozilla/5.0 (compatible; uMBot-LN/1.0; mailto: crawling@ubermetrics-technologies.com)": true,
|
||||||
"Mozilla/5.0 (compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)": true,
|
"Mozilla/5.0 (compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)": true,
|
||||||
"Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)": true,
|
"Mozilla/5.0+(compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)": true,
|
||||||
"Mozilla/5.0 (compatible; woriobot +http://worio.com)": true,
|
"Mozilla/5.0 (compatible; woriobot +http://worio.com)": true,
|
||||||
"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)": true,
|
"Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)": true,
|
||||||
"Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)": true,
|
"Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)": true,
|
||||||
"Mozilla/5.0 (compatible; Yeti/1.1; +http://help.naver.com/robots/)": true,
|
"Mozilla/5.0 (compatible; Yeti/1.1; +http://help.naver.com/robots/)": true,
|
||||||
"Mozilla/5.0 (compatible; zitebot support [at] zite [dot] com +http://zite.com)": true,
|
"Mozilla/5.0 (compatible; zitebot support [at] zite [dot] com +http://zite.com)": true,
|
||||||
"Mozilla/5.0/Firefox/42.0 (contactbigdatafr at gmail.com)": true,
|
"Mozilla/5.0/Firefox/42.0 (contactbigdatafr at gmail.com)": true,
|
||||||
"Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)": true,
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Applebot/0.3; +http://www.apple.com/go/applebot)": true,
|
||||||
"Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true,
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5376e Safari/8536.25 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)": true,
|
||||||
"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)": true,
|
"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; http://www.bing.com/bingbot.htm)": true,
|
||||||
@@ -176,13 +176,13 @@ var bots = map[string]bool{
|
|||||||
"Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.1.3) Gecko/20090914 Slackware/13.0_stable Firefox/3.5.3": true,
|
"Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.1.3) Gecko/20090914 Slackware/13.0_stable Firefox/3.5.3": true,
|
||||||
"Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9a1) Gecko/20060112 Firefox/1.6a1": true,
|
"Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9a1) Gecko/20060112 Firefox/1.6a1": true,
|
||||||
"Mozilla/5.0 (X11; U; Linux x86_64; zh-TW; rv:1.9.0.13) Gecko/2009080315 Ubuntu/9.04 (jaunty) Firefox/3.0.13": true,
|
"Mozilla/5.0 (X11; U; Linux x86_64; zh-TW; rv:1.9.0.13) Gecko/2009080315 Ubuntu/9.04 (jaunty) Firefox/3.0.13": true,
|
||||||
"myspider/Nutch-1.10": true,
|
"myspider/Nutch-1.10": true,
|
||||||
"netEstate NE Crawler (+http://www.website-datenbank.de/)": true,
|
"netEstate NE Crawler (+http://www.website-datenbank.de/)": true,
|
||||||
"omgili/0.5 +http://omgili.com": true,
|
"omgili/0.5 +http://omgili.com": true,
|
||||||
"PercolateCrawler/4 (ops@percolate.com)": true,
|
"PercolateCrawler/4 (ops@percolate.com)": true,
|
||||||
"python-requests/2.6.2 CPython/2.7.6 Linux/3.13.0-43-generic": true,
|
"python-requests/2.6.2 CPython/2.7.6 Linux/3.13.0-43-generic": true,
|
||||||
"python-requests/2.6.2 CPython/2.7.6 Linux/3.13.0-61-generic": true,
|
"python-requests/2.6.2 CPython/2.7.6 Linux/3.13.0-61-generic": true,
|
||||||
"Python-urllib/2.7": true,
|
"Python-urllib/2.7": true,
|
||||||
"rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+shiny@moz.com)": true,
|
"rogerbot/1.0 (http://moz.com/help/pro/what-is-rogerbot-, rogerbot-crawler+shiny@moz.com)": true,
|
||||||
"Ruby": true,
|
"Ruby": true,
|
||||||
"SafeDNSBot (https://www.safedns.com/searchbot)": true,
|
"SafeDNSBot (https://www.safedns.com/searchbot)": true,
|
||||||
@@ -202,7 +202,7 @@ var bots = map[string]bool{
|
|||||||
"Stratagems Kumo": true,
|
"Stratagems Kumo": true,
|
||||||
"Traackr.com Bot": true,
|
"Traackr.com Bot": true,
|
||||||
"voltron": true,
|
"voltron": true,
|
||||||
"Wotbox/2.01 (+http://www.wotbox.com/bot/)": true,
|
"Wotbox/2.01 (+http://www.wotbox.com/bot/)": true,
|
||||||
"yacybot (freeworld/global; amd64 Linux 3.12.43-52.6-default; java 1.8.0_40; Europe/en) http://yacy.net/bot.html": true,
|
"yacybot (freeworld/global; amd64 Linux 3.12.43-52.6-default; java 1.8.0_40; Europe/en) http://yacy.net/bot.html": true,
|
||||||
"yacybot (/global; x86_64 Mac OS X 10.11.4; java 1.8.0_77; America/en) http://yacy.net/bot.html": true,
|
"yacybot (/global; x86_64 Mac OS X 10.11.4; java 1.8.0_77; America/en) http://yacy.net/bot.html": true,
|
||||||
"YisouSpider": true,
|
"YisouSpider": true,
|
||||||
|
|||||||
Reference in New Issue
Block a user