ApacheSite: install ultimate bad bot blocker

Currently we use manual blacklisting rules to block bad bots/malicious scans
on our apache installs.  This has led to a situation where not all servers
are in sync, and adding new rules is tedious.

This change installed the Ultimate Bad Blocker configs to allow us
a centralized way to handle this.  More info can be found at:

https://github.com/mitchellkrogza/apache-ultimate-bad-bot-blocker/tree/master/Apache_2.4

See SYS-143.

Change-Id: I2ba1e7d8efe8507cc9fe5c1f823b7f5c1fbb8b21
Reviewed-on: https://review.linaro.org/29544
Reviewed-by: Benjamin Copeland <ben.copeland@linaro.org>
Reviewed-by: Riku Voipio <riku.voipio@linaro.org>

7 files changed, 385 insertions, 0 deletions

diff --git a/roles/apache-site/files/bad-referrer-words.conf b/roles/apache-site/files/bad-referrer-words.conf
new file mode 100644
index 00000000..8a55462f
--- /dev/null
+++ b/roles/apache-site/files/bad-referrer-words.conf
@@ -0,0 +1,70 @@
+# EDIT THIS FILE AS YOU LIKE TO ADD OR REMOVE ANY BAD REFERRERS or BAD WORDS YOU WANT TO SCAN FOR ###
+
+##############################################################################
+#        ___                 __                                              #
+#       / _ | ___  ___ _____/ /  ___                                         #
+#      / __ |/ _ \/ _ `/ __/ _ \/ -_)                                        #
+#     /_/ |_/ .__/\_,_/\__/_//_/\__/                                         #
+#        __/_/        __   ___       __     ___  __         __               #
+#       / _ )___ ____/ /  / _ )___  / /_   / _ )/ /__  ____/ /_____ ____     #
+#      / _  / _ `/ _  /  / _  / _ \/ __/  / _  / / _ \/ __/  '_/ -_) __/     #
+#     /____/\_,_/\_,_/  /____/\___/\__/  /____/_/\___/\__/_/\_\\__/_/        #
+#                                                                            #
+##############################################################################
+
+# This is merely an example and gets auto included as since Version 2.2017.07 introduced on 2017-04-20
+# This file must exist on your system or Apache will fail a reload due to a missing file
+# For all intensive purpose you can delete everything inside this file and leave it
+# completely blank if you do not want your Apache Blocker to include scanning for bad words within urls or referrer string
+# Only add one entry per line
+
+# PLEASE NOTE:
+# THIS INCLUDE FILES IS TO BE USED FOR CREATING YOUR OWN CUSTOM SET OF BAD REFERRERS AS WELL AS BAD REFERRER WORD SCANNING
+
+# BY DEFAULT ALL THE EXAMPLES BELOW ARE COMMENTED OUT AND HENCE NOT ENABLED
+
+
+	# *************************
+	# CUSTOM BAD REFERRERS HERE
+	# *************************
+
+
+	#SetEnvIfNoCase Referer ~*somebad\.website spam_ref
+	#SetEnvIfNoCase Referer ~*somethingbad\.com spam_ref
+	#SetEnvIfNoCase Referer ~*veryverbad\.com spam_ref
+	#SetEnvIfNoCase Referer ~*superbadwebsite\.com spam_ref
+
+
+
+	# *******************************
+	# CUSTOM BAD REFERRERS WORDS HERE
+	# *******************************
+
+	# *******************************
+	# !!! WARNING WARNING WARNING !!!
+	# *******************************
+
+	# ***************************************
+	# PLEASE BE VERY CAREFUL HOW YOU USE THIS
+	# ***************************************
+
+	# Here is an example of how one supposed bad word can cause your whole site to go down.
+	# An issue was logged where the users own domain name was specialisteparquet.com
+	# Because this list contained the word "cialis" it was detected within his domain name causing 
+	# his entire site to go down and not server any assets.
+	# That one entry would even cause any site containing a word like "specialist" anywhere in any
+	# of their sites pages to cause them to be blocked and whitelisting your own domain name in the
+	# whitelist-domains.conf file will not even bypass this, SO BE CAREFUL PLEASE
+
+	#SetEnvIfNoCase Referer ~*adultgalls spam_ref
+	#SetEnvIfNoCase Referer ~*bigblackbooty spam_ref
+	#SetEnvIfNoCase Referer ~*cookie-law-enforcement spam_ref
+	#SetEnvIfNoCase Referer ~*free-share-buttons spam_ref
+	#SetEnvIfNoCase Referer ~*free-social-buttons spam_ref
+	#SetEnvIfNoCase Referer ~*fuck-paid-share-buttons spam_ref
+	#SetEnvIfNoCase Referer ~*law-enforcement-bot spam_ref
+	#SetEnvIfNoCase Referer ~*law-enforcement-check spam_ref
+	#SetEnvIfNoCase Referer ~*share-buttons-for-free spam_ref
+	#SetEnvIfNoCase Referer ~*social-buttons- spam_ref
+	#SetEnvIfNoCase Referer ~*vvakhrin-ws1 spam_ref
+	#SetEnvIfNoCase Referer ~*xxxrus spam_ref
diff --git a/roles/apache-site/files/blacklist-ips.conf b/roles/apache-site/files/blacklist-ips.conf
new file mode 100644
index 00000000..430d1627
--- /dev/null
+++ b/roles/apache-site/files/blacklist-ips.conf
@@ -0,0 +1,183 @@
+# EDIT THIS FILE AS YOU LIKE TO ADD OR REMOVE ANY BAD IP ADDRESSES OR IP RANGES YOU WANT TO BLOCK ###
+
+##############################################################################
+#        ___                 __                                              #
+#       / _ | ___  ___ _____/ /  ___                                         #
+#      / __ |/ _ \/ _ `/ __/ _ \/ -_)                                        #
+#     /_/ |_/ .__/\_,_/\__/_//_/\__/                                         #
+#        __/_/        __   ___       __     ___  __         __               #
+#       / _ )___ ____/ /  / _ )___  / /_   / _ )/ /__  ____/ /_____ ____     #
+#      / _  / _ `/ _  /  / _  / _ \/ __/  / _  / / _ \/ __/  '_/ -_) __/     #
+#     /____/\_,_/\_,_/  /____/\___/\__/  /____/_/\___/\__/_/\_\\__/_/        #
+#                                                                            #
+##############################################################################
+
+# This is merely an example and gets auto included as since Version 2.2017.05 introduced on 2017-04-19
+# This file must exist on your system or Apache will fail a reload due to a missing file
+# For all intensive purpose you can delete everything inside this file and leave it
+# completely blank if you do not want your Apache Blocker to do any blocking of bad IP's
+
+Require not ip 104.223.37.150
+Require not ip 104.5.92.27
+Require not ip 107.150.63.170
+Require not ip 109.236.83.247
+Require not ip 137.74.49.205
+Require not ip 137.74.49.208
+Require not ip 146.0.74.150
+Require not ip 148.251.54.44
+Require not ip 149.56.151.180
+Require not ip 149.56.232.146
+Require not ip 150.70.0.0/16
+Require not ip 151.80.27.90
+Require not ip 151.80.99.90
+Require not ip 151.80.99.91
+Require not ip 154.16.199.144
+Require not ip 154.16.199.34
+Require not ip 154.16.199.48
+Require not ip 154.16.199.78
+Require not ip 158.69.142.34
+Require not ip 166.62.80.172
+Require not ip 173.212.192.219
+Require not ip 173.234.11.105
+Require not ip 173.234.153.106
+Require not ip 173.234.153.30
+Require not ip 173.234.175.68
+Require not ip 173.234.31.9
+Require not ip 173.234.38.25
+Require not ip 176.126.245.213
+Require not ip 178.238.234.1
+Require not ip 185.35.63.128
+Require not ip 185.100.87.238
+Require not ip 185.115.125.99
+Require not ip 185.119.81.11
+Require not ip 185.119.81.63
+Require not ip 185.119.81.77
+Require not ip 185.119.81.78
+Require not ip 185.130.225.65
+Require not ip 185.130.225.66
+Require not ip 185.130.225.83
+Require not ip 185.130.225.90
+Require not ip 185.130.225.94
+Require not ip 185.130.225.95
+Require not ip 185.130.226.105
+Require not ip 185.153.197.103
+Require not ip 185.159.36.6
+Require not ip 185.183.96.33
+Require not ip 185.47.62.199
+Require not ip 185.62.190.38
+Require not ip 185.70.105.161
+Require not ip 185.70.105.164
+Require not ip 185.85.239.156
+Require not ip 185.85.239.157
+Require not ip 185.86.13.213
+Require not ip 185.86.5.199
+Require not ip 185.86.5.212
+Require not ip 185.92.72.88
+Require not ip 185.93.185.11
+Require not ip 185.93.185.12
+Require not ip 188.209.52.101
+Require not ip 190.152.223.27
+Require not ip 191.96.249.29
+Require not ip 192.69.89.173
+Require not ip 193.201.224.205
+Require not ip 195.154.183.190
+Require not ip 195.229.241.174
+Require not ip 200.7.105.43
+Require not ip 210.212.194.60
+Require not ip 216.218.147.194
+Require not ip 220.227.234.129
+Require not ip 23.253.230.158
+Require not ip 23.89.159.176
+Require not ip 31.170.160.209
+Require not ip 45.32.186.11
+Require not ip 45.76.21.179
+Require not ip 46.249.38.145
+Require not ip 46.249.38.146
+Require not ip 46.249.38.148
+Require not ip 46.249.38.149
+Require not ip 46.249.38.150
+Require not ip 46.249.38.151
+Require not ip 46.249.38.152
+Require not ip 46.249.38.153
+Require not ip 46.249.38.154
+Require not ip 46.249.38.159
+Require not ip 51.255.172.22
+Require not ip 5.39.218.232
+Require not ip 5.39.219.24
+Require not ip 5.39.222.18
+Require not ip 5.39.223.134
+Require not ip 54.213.16.154
+Require not ip 54.213.9.111
+Require not ip 62.210.146.49
+Require not ip 62.210.88.4
+Require not ip 65.98.91.181
+Require not ip 69.162.124.237
+Require not ip 69.64.147.24
+Require not ip 72.8.183.202
+Require not ip 77.247.178.191
+Require not ip 77.247.178.47
+Require not ip 77.247.181.219
+Require not ip 78.31.184.0/21
+Require not ip 78.31.211.0/24
+Require not ip 79.110.128.17
+Require not ip 79.110.128.63
+Require not ip 79.110.128.252
+Require not ip 79.110.128.128
+Require not ip 80.87.205.10
+Require not ip 80.87.205.11
+Require not ip 85.17.230.23
+Require not ip 85.17.26.68
+Require not ip 91.185.190.172
+Require not ip 91.200.12.0/22
+Require not ip 91.200.12.15
+Require not ip 91.200.12.49
+Require not ip 91.200.12.91
+Require not ip 92.222.66.137
+Require not ip 93.104.209.11
+Require not ip 93.158.200.103
+Require not ip 93.158.200.105
+Require not ip 93.158.200.115
+Require not ip 93.158.200.124
+Require not ip 93.158.200.126
+Require not ip 93.158.200.66
+Require not ip 93.158.200.68
+Require not ip 93.238.202.44
+
+# Cyveillance / Qwest Communications / PSINET
+# *******************************************
+# I am extensively researching this subject - appears to be US government involved
+# and also appears to be used by all sorts of law enforcement agencies. For one they 
+# do not obey robots.txt and continually disguise their User-Agent strings. Time will
+# tell if this is all correct or not.
+# For now see - https://en.wikipedia.org/wiki/Cyveillance
+
+# IMPORTANT UPDATE ON Cyveillance / Qwest Communications !!!
+# **********************************************************
+# I have done a lot of research on Cyveillance now and through monitoring my logs I know
+# for sure what companies are using them and what they are actually looking for.
+# My research has led me to understand that Cyveillance services are used by hundreds
+# of companies to help them dicsover theft of copyrighted materials like images, movies
+# music and other materials. I personally believe a lot of block lists who originally recommended
+# blocking Cyveillance have done so to protect their torrent or p2p sites from being scanned.
+# I personally have now unblocked them as image theft is a big problem of mine but if you
+# do want to allow Cyveillance you can simply modify the entries in the below from "Require not ip" to "Require ip"
+# Getty Images is one such company who appears to use Cyveillance to help monitor for copyright theft.
+  
+# Use this section at YOUR OWN RISK, you may block some legitimate networks but after many hours of
+# Research this is now the completely updated list of all IP ranges IPV4 and IPV6 owned Qwest Communications
+# PSINET and Cyveillance
+
+# IMPORTANT NOTE: If you really want to keeps bot and things out of certain parts of your web site
+# Rather implement a comlex Google Re-Captcha to reach sections of your sites and for people to be able
+# to access download links. Google Re-Captcha with images is too complex for any bot.
+
+Require not ip 4.17.135.32/27
+Require not ip 38.0.0.0/8
+Require not ip 63.144.0.0/13
+Require not ip 65.112.0.0/12
+Require not ip 65.192.0.0/11
+Require not ip 65.213.208.128/27
+Require not ip 65.222.176.96/27
+Require not ip 65.222.185.72/29
+Require not ip 206.2.138.0/23
+Require not ip 208.71.164.0/22
diff --git a/roles/apache-site/files/blacklist-user-agents.conf b/roles/apache-site/files/blacklist-user-agents.conf
new file mode 100644
index 00000000..e146f40e
--- /dev/null
+++ b/roles/apache-site/files/blacklist-user-agents.conf
@@ -0,0 +1,60 @@
+# EDIT THIS FILE AS YOU LIKE TO OVER-RIDE, BLACKLIST OR WHITELIST 
+# ANY BAD USER-AGENT STRINGS YOU WANT TO SCAN FOR
+# ****************************************************************************
+
+##############################################################################
+#        ___                 __                                              #
+#       / _ | ___  ___ _____/ /  ___                                         #
+#      / __ |/ _ \/ _ `/ __/ _ \/ -_)                                        #
+#     /_/ |_/ .__/\_,_/\__/_//_/\__/                                         #
+#        __/_/        __   ___       __     ___  __         __               #
+#       / _ )___ ____/ /  / _ )___  / /_   / _ )/ /__  ____/ /_____ ____     #
+#      / _  / _ `/ _  /  / _  / _ \/ __/  / _  / / _ \/ __/  '_/ -_) __/     #
+#     /____/\_,_/\_,_/  /____/\___/\__/  /____/_/\___/\__/_/\_\\__/_/        #
+#                                                                            #
+##############################################################################
+
+# Add One Entry Per Line - List all the extra bad User-Agents you want to permanently block or whitelist.
+# You can also use this include file to over-ride user-agents like wget or curl which are previously
+# white-listed in the main list. This gives you full control over what you want to allow access.
+
+# This is for additional User-Agents that are not included in the main list of the bot blocker
+# This is also used to over-ride User-Agents in the main list
+
+# This file must exist on your system or Apache will fail a reload due to a missing file
+# This allows you finer control of keeping certain bots blocked and automatic updates will
+# Never be able to remove this custom list of yours 
+
+# Please note this include file loads first before any of the already whitelisted User-Agents
+# in the bad bot blocker. By loading first in line it over-rides anything below it so for instance
+# if you want to block Baidu, Google or Bing for any reason you add them to this file which loads
+# first and takes precedence over anything below it. This now allows even finer control over the 
+# bad bot blocker. Enjoy !!!
+
+# Even though this file is called blacklist-user-agents, as mentioned it can also be used to whitelist user agents
+# By adding them below and setting the word bad_bot to good_bot this will permanently whitelist the User-Agent.
+
+# Make sure any words that contain special characters are escaped and include word boundaries as per the Regex examples below.
+
+# EXAMPLE 1: the User-Agent name "someverybaduseragentname1" is entered as "\bsomeverybaduseragentname1\b"
+# EXAMPLE 2: the User-Agent name "some-very-bad-useragentname2" is entered as "\bsome\-very\-bad\-useragentname1\b" 
+# EXAMPLE 3: if you want to block something like wget you would add the following "\bwget\b"
+
+# the "\b" are word boundaries which prevents partial matching and false positives.
+# Follow the regex formatting examples below.
+
+# BY DEFAULT ALL THE EXAMPLES BELOW ARE COMMENTED OUT AND HENCE NOT ENABLED
+
+	# *************************
+	# BLACKLIST ADDITIONAL BOTS
+	# *************************
+
+	#BrowserMatchNoCase "^(.*?)(\bMyVeryBadUserAgentName\b)(.*)$" bad_bot
+	#BrowserMatchNoCase "^(.*?)(\bMy\ Bad\ User\ Agent\b)(.*)$" bad_bot
+
+	# *************************
+	# WHITELIST ADDITIONAL BOTS
+	# *************************
+
+	#BrowserMatchNoCase "^(.*?)(\bMyVeryGoodUserAgentName\b)(.*)$" good_bot
+	#BrowserMatchNoCase "^(.*?)(\bMy\ Bad\ User\ Agent\b)(.*)$" good_bot
diff --git a/roles/apache-site/files/whitelist-domains.conf b/roles/apache-site/files/whitelist-domains.conf
new file mode 100644
index 00000000..d1cb8a2a
--- /dev/null
+++ b/roles/apache-site/files/whitelist-domains.conf
@@ -0,0 +1,27 @@
+# EDIT THIS FILE AS YOU LIKE TO WHITELIST YOUR OWN DOMAIN NAMES AND SPARE THEM FROM ANY REFERRER CHECKING ###
+
+##############################################################################
+#        ___                 __                                              #
+#       / _ | ___  ___ _____/ /  ___                                         #
+#      / __ |/ _ \/ _ `/ __/ _ \/ -_)                                        #
+#     /_/ |_/ .__/\_,_/\__/_//_/\__/                                         #
+#        __/_/        __   ___       __     ___  __         __               #
+#       / _ )___ ____/ /  / _ )___  / /_   / _ )/ /__  ____/ /_____ ____     #
+#      / _  / _ `/ _  /  / _  / _ \/ __/  / _  / / _ \/ __/  '_/ -_) __/     #
+#     /____/\_,_/\_,_/  /____/\___/\__/  /____/_/\___/\__/_/\_\\__/_/        #
+#                                                                            #
+##############################################################################
+
+# Add One Entry Per Line - List all your own domains of the sites you host on the server
+# This file must exist on your system or Nginx will fail a reload due to a missing file
+# Automatic updates will never be able to remove this custom list of yours 
+# Add One Entry Per Line
+
+# Make sure any domains have dots and special characters escaped as per the Regex examples below.
+# For example myfirstowndomainname.com should be entered as myfirstowndomainname\.com
+# and my-second-owndomainname.com should be entered as my\-second\-owndomainname\.com
+
+# BY DEFAULT ALL THE EXAMPLES BELOW ARE COMMENTED OUT AND HENCE NOT ENABLED
+
+	#SetEnvIfNoCase Referer ~*yourdomain\.com good_ref
+	#SetEnvIfNoCase Referer ~*your\-domain\.com good_ref
diff --git a/roles/apache-site/files/whitelist-ips.conf b/roles/apache-site/files/whitelist-ips.conf
new file mode 100644
index 00000000..4e49309b
--- /dev/null
+++ b/roles/apache-site/files/whitelist-ips.conf
@@ -0,0 +1,20 @@
+# Add One Entry Per Line
+
+##############################################################################
+#        ___                 __                                              #
+#       / _ | ___  ___ _____/ /  ___                                         #
+#      / __ |/ _ \/ _ `/ __/ _ \/ -_)                                        #
+#     /_/ |_/ .__/\_,_/\__/_//_/\__/                                         #
+#        __/_/        __   ___       __     ___  __         __               #
+#       / _ )___ ____/ /  / _ )___  / /_   / _ )/ /__  ____/ /_____ ____     #
+#      / _  / _ `/ _  /  / _  / _ \/ __/  / _  / / _ \/ __/  '_/ -_) __/     #
+#     /____/\_,_/\_,_/  /____/\___/\__/  /____/_/\___/\__/_/\_\\__/_/        #
+#                                                                            #
+##############################################################################
+
+# BY DEFAULT ALL THE EXAMPLES BELOW ARE COMMENTED OUT AND HENCE NOT ENABLED
+
+	#Require ip 192.168.1.0 
+
+
+# DO NOT EVER USE 127.0.0.1 only real public facing IP addresses.
\ No newline at end of file
diff --git a/roles/apache-site/tasks/block_bad_bots.yml b/roles/apache-site/tasks/block_bad_bots.yml
new file mode 100644
index 00000000..30ff0f22
--- /dev/null
+++ b/roles/apache-site/tasks/block_bad_bots.yml
@@ -0,0 +1,21 @@
+---
+- name: Create apache2 custom config directory
+  file: path=/etc/apache2/custom.d state=directory
+
+- name: Install global blacklist
+  get_url:
+    url:  https://raw.githubusercontent.com/mitchellkrogza/apache-ultimate-bad-bot-blocker/master/Apache_2.4/custom.d/globalblacklist.conf
+    dest: /etc/apache2/custom.d/globalblacklist.conf
+  notify:
+    - reload-apache
+
+- name: Install custom lists
+  copy:
+    src: files/{{item}}
+    dest: /etc/apache2/custom.d/{{item}}
+  with_items:
+    - whitelist-ips.conf
+    - whitelist-domains.conf
+    - blacklist-ips.conf
+    - blacklist-user-agents.conf
+    - bad-referrer-words.conf
diff --git a/roles/apache-site/tasks/main.yml b/roles/apache-site/tasks/main.yml
index 21eac766..14118580 100644
--- a/roles/apache-site/tasks/main.yml
+++ b/roles/apache-site/tasks/main.yml
@@ -23,3 +23,7 @@
       - reload-apache
   tags:
       - apache
+
+- name: Install Block Bad Bots configurations
+  include_tasks: block_bad_bots.yml
+  when: block_bad_bots is defined