grep-v-web-bots

#!/bin/sh
set -euf -o pipefail

##
# grep without web robot strings like robots.txt, Googlebot, Mouseflow, etc.
#
# This is useful e.g. for grepping web log files.
#
# Contact: Joel Parker Henderson (joel@joelparkerhenderson.com)
# License: GPL
# Updated: 2015-01-25
##

grep -v "\b\(robots.txt\|Baiduspider\|bingbot\|COMODOspider\|Googlebot\|Mouseflow\|Sosospider\)\b"