Explain findBots.sh a little more

This commit is contained in:
Matt Baer
2018-01-25 17:29:58 -05:00
parent d38bfb4ae9
commit ca03ff920a
+4 -1
View File
@@ -1,7 +1,10 @@
#!/bin/bash #!/bin/bash
# #
# Generates a Go map containing all bots that have accessed Write.as # Generates a Go map containing all bots that have accessed Write.as from the
# application logs stored in /var/log/
#
# usage: findBots.sh application.log
# #
cat /var/log/$1 | grep -i 'bot\|spider\|crawl\|scraper\|indexer\|voltron' | awk -F\" '{print $4}' | sort | uniq > bots.txt cat /var/log/$1 | grep -i 'bot\|spider\|crawl\|scraper\|indexer\|voltron' | awk -F\" '{print $4}' | sort | uniq > bots.txt