Add spiders, scrapers, indexers; update bots

2015-08-17 15:21:43 -04:00
parent 2054237757
commit da1d3d898c
2 changed files with 20 additions and 7 deletions
@@ -4,7 +4,7 @@
 # Generates a Go map containing all bots that have accessed Write.as
 #

-cat /var/log/$1 | grep -i 'bot\|crawler\|voltron' | awk -F\" '{print $4}' | sort | uniq > bots.txt
+cat /var/log/$1 | grep -i 'bot\|spider\|crawl\|scraper\|indexer\|voltron' | awk -F\" '{print $4}' | sort | uniq > bots.txt

 rm bots.go