apache: centralize a list of bad user agents

This change centralizes our list of bad user agents
and puts it into one file to be maintained across
all of our websites.

Change-Id: I0e24b6b9713ac7eeed957a5b184303371f9cc485
Signed-off-by: Kelley Spoon <kelley.spoon@linaro.org>
Reviewed-on: https://review.linaro.org/c/infrastructure/ansible-playbooks/+/46189
Reviewed-by: Benjamin Copeland <ben.copeland@linaro.org>

14 files changed, 61 insertions, 11 deletions

diff --git a/files/apache/android-git.linaro.org.conf b/files/apache/android-git.linaro.org.conf
index 03bab4db..7f09ed40 100644
--- a/files/apache/android-git.linaro.org.conf
+++ b/files/apache/android-git.linaro.org.conf
@@ -11,6 +11,8 @@ Mutex default rewrite-map
     RewriteEngine on
     RewriteCond %{REQUEST_URI} !^/.well-known/acme-challenge/
     RewriteRule ^(.*)$ https://{{ git_host }}$1 [R=301,L]
+
+    Include /etc/apache2/linaro/block-refs.conf
 </VirtualHost>
 
 {% if inventory_hostname == 'android-git-us.linaro.org' %}
@@ -100,6 +102,8 @@ Mutex default rewrite-map
     #RewriteLog ${APACHE_LOG_DIR}/{{ git_host }}-rewrite.log
     #RewriteLogLevel 0
 
+    Include /etc/apache2/linaro/block-refs.conf
+
     RewriteMap gitdirs prg:{{tools_checkout_dir}}/linaro-git-tools/git-repo-url-rewrite/git-directory-rewritemap.py
     RewriteRule ^/git-ro/(.*) /git-http/${gitdirs:$1} [P,L]
 
diff --git a/files/apache/android-review.linaro.org.conf b/files/apache/android-review.linaro.org.conf
index 6eacf234..340ea3dc 100644
--- a/files/apache/android-review.linaro.org.conf
+++ b/files/apache/android-review.linaro.org.conf
@@ -48,7 +48,9 @@
 	ProxyPass	/	http://127.0.0.1:8080/ nocanon
 	{% endif %}
 
-        Include /etc/apache2/linaro/letsencrypt.conf
+    RewriteEngine On
+    Include /etc/apache2/linaro/block-refs.conf
+    Include /etc/apache2/linaro/letsencrypt.conf
 </VirtualHost>
 
 {% if ssl_cert is defined %}
@@ -102,5 +104,8 @@
 
 	AllowEncodedSlashes On
 	ProxyPass	/	http://127.0.0.1:8080/ nocanon
+
+    RewriteEngine On
+    Include /etc/apache2/linaro/block-refs.conf
 </VirtualHost>
 {% endif %}
diff --git a/files/apache/dev-private-git.linaro.org.conf b/files/apache/dev-private-git.linaro.org.conf
index 5f0a803b..7c5aeba6 100644
--- a/files/apache/dev-private-git.linaro.org.conf
+++ b/files/apache/dev-private-git.linaro.org.conf
@@ -19,6 +19,9 @@ LDAPOpCacheTTL 36000
 
     RedirectMatch permanent "^/(?!\.well-known/acme-challenge)(.*)" "https://{{git_host}}/$1"
     Include /etc/apache2/linaro/letsencrypt.conf
+
+    RewriteEngine On
+    Include /etc/apache2/linaro/block-refs.conf
 </VirtualHost>
 
 # Support for deprecated *.git.linaro.org subdomains
@@ -27,6 +30,9 @@ LDAPOpCacheTTL 36000
     ServerAlias zte.git.linaro.org
 
     RewriteEngine On
+
+    Include /etc/apache2/linaro/block-refs.conf
+
     RewriteCond %{HTTP_HOST} ^zte.git.linaro.org [nocase]
     RewriteRule ^(.*) https://zte-git.linaro.org$1 [redirect=301,noescape,last]
     RewriteRule ^(.*) https://{{ git_host }}$1 [redirect=301,noescape,last]
@@ -134,4 +140,5 @@ LDAPOpCacheTTL 36000
         Options ExecCGI FollowSymlinks
         Require all granted
     </Directory>
+
 </VirtualHost>
diff --git a/files/apache/dev-private-review.linaro.org.conf b/files/apache/dev-private-review.linaro.org.conf
index 5bee707b..7011eb1f 100644
--- a/files/apache/dev-private-review.linaro.org.conf
+++ b/files/apache/dev-private-review.linaro.org.conf
@@ -52,6 +52,9 @@ LDAPOpCacheTTL 36000
     AllowEncodedSlashes On
     ProxyPass / http://127.0.0.1:8080/ nocanon
     {% endif %}
+
+    RewriteEngine On
+    Include /etc/apache2/linaro/block-refs.conf
 </VirtualHost>
 
 {% if ssl_cert is defined %}
@@ -96,6 +99,7 @@ LDAPOpCacheTTL 36000
     </Location>
 
     RewriteEngine On
+    Include /etc/apache2/linaro/block-refs.conf
     RewriteCond %{HTTP_COOKIE} !\bGerritAccount\b
     RewriteCond %{REQUEST_URI} /c/
     RewriteCond %{REQUEST_URI} !/login
diff --git a/files/apache/git.linaro.org.conf b/files/apache/git.linaro.org.conf
index d2345c0d..b3fc8429 100644
--- a/files/apache/git.linaro.org.conf
+++ b/files/apache/git.linaro.org.conf
@@ -64,8 +64,7 @@ ServerTokens Prod
 
     AllowEncodedSlashes On
 
-    RewriteCond %{HTTP_USER_AGENT} (AhrefsBot|amazonbot|bingbot|Baidu|Baiduspider|360Spider|360|^MauiBot|^SemrushBot|^MegaIndex|PetalBot) [nocase]
-    RewriteRule ^(.*)$ - [forbidden,last]
+    Include /etc/apache2/linaro/block-refs.conf
 
     RewriteCond %{REQUEST_URI} ^/jmx-console(.*)$
     RewriteRule ^/(.*)$ - [forbidden,last]
@@ -203,8 +202,7 @@ ServerTokens Prod
     #RewriteLog ${APACHE_LOG_DIR}/{{ git_host }}-rewrite.log
     #RewriteLogLevel 0
 
-    RewriteCond %{HTTP_USER_AGENT} (AhrefsBot|amazonbot|bingbot|Baidu|Baiduspider|360Spider|360|SemrushBot|PetalBot) [nocase]
-    RewriteRule ^(.*)$ - [forbidden,last]
+    Include /etc/apache2/linaro/block-refs.conf
 
     RewriteRule ^/landing-teams/working/qualcomm/(.*).git(.*) https://git.codelinaro.org/linaro/qcomlt/$1 [R,L]
 
diff --git a/files/apache/grafana.linaro.org.conf b/files/apache/grafana.linaro.org.conf
index c4f659c8..9653a310 100644
--- a/files/apache/grafana.linaro.org.conf
+++ b/files/apache/grafana.linaro.org.conf
@@ -17,6 +17,9 @@
 {% endif %}
     ErrorLog ${APACHE_LOG_DIR}/{{inventory_hostname}}-error.log
     CustomLog ${APACHE_LOG_DIR}/{{inventory_hostname}}-access.log combined
+
+    RewriteEngine On
+    Include /etc/apache2/linaro/block-refs.conf
 </VirtualHost>
 
 <IfModule mod_ssl.c>
@@ -32,6 +35,9 @@
     ServerAlias {{apache_aliases|join(' ')}}
 {% endif %}
 
+    RewriteEngine On
+    Include /etc/apache2/linaro/block-refs.conf
+
     Include /etc/apache2/linaro/settings-ssl.conf
     SSLCertificateFile {{ssl_cert}}
     SSLCertificateKeyFile {{ssl_key}}
diff --git a/files/apache/review.linaro.org.conf b/files/apache/review.linaro.org.conf
index 46512154..cd726387 100644
--- a/files/apache/review.linaro.org.conf
+++ b/files/apache/review.linaro.org.conf
@@ -15,6 +15,7 @@
 
     AllowEncodedSlashes On
     RewriteEngine On
+    Include /etc/apache2/linaro/block-refs.conf
     RewriteCond %{REQUEST_URI} !^/.well-known/acme-challenge/
     RewriteCond %{HTTP_USER_AGENT} (AhrefsBot|amazonbot|bingbot|Baidu|Baiduspider|360Spider|360|^MauiBot|^SemrushBot|^MegaIndex|PetalBot) [nocase]
     RewriteRule ^(.*)$ - [forbidden,last]
@@ -59,6 +60,7 @@
 
         AllowEncodedSlashes On
         RewriteEngine On
+        Include /etc/apache2/linaro/block-refs.conf
         RewriteCond %{HTTP_USER_AGENT} (AhrefsBot|amazonbot|bingbot|Baidu|Baiduspider|360Spider|360|SemrushBot|PetalBot) [nocase]
         RewriteRule ^(.*)$ - [forbidden,last]
 
diff --git a/files/apache/review.mlplatform.org.conf b/files/apache/review.mlplatform.org.conf
index 6258f214..9652d72a 100644
--- a/files/apache/review.mlplatform.org.conf
+++ b/files/apache/review.mlplatform.org.conf
@@ -9,6 +9,9 @@
 
     RedirectMatch permanent "^/(?!\.well-known/acme-challenge)(.*)" "https://{{hostname}}/$1"
 
+    RewriteEngine On
+    Include /etc/apache2/linaro/block-refs.conf
+
     Include /etc/apache2/linaro/letsencrypt.conf
 </VirtualHost>
 
@@ -32,6 +35,9 @@
         Header set Cache-Control "max-age=86400, public"
     </FilesMatch>
 
+    RewriteEngine On
+    Include /etc/apache2/linaro/block-refs.conf
+
     ProxyRequests Off
     ProxyVia Off
     ProxyPreserveHost On
diff --git a/files/apache/review.trustedfirmware.org.conf b/files/apache/review.trustedfirmware.org.conf
index c137f1fd..b4b5f546 100644
--- a/files/apache/review.trustedfirmware.org.conf
+++ b/files/apache/review.trustedfirmware.org.conf
@@ -9,6 +9,8 @@
 
     RedirectMatch permanent "^/(?!\.well-known/acme-challenge)(.*)" "https://{{hostname}}/$1"
     Include /etc/apache2/linaro/letsencrypt.conf
+    RewriteEngine On
+    Include /etc/apache2/linaro/block-refs.conf
 </VirtualHost>
 
 <VirtualHost *:443>
@@ -31,6 +33,7 @@
     </FilesMatch>
 
     RewriteEngine on
+    Include /etc/apache2/linaro/block-refs.conf
     RewriteRule "^/c/trusted-firmware-m(.*)" "/c/TF-M/trusted-firmware-m$1" [redirect=301]
 
     ProxyRequests Off
diff --git a/files/apache/snapshots.linaro.org.conf b/files/apache/snapshots.linaro.org.conf
index 0a13c24d..cf24465b 100644
--- a/files/apache/snapshots.linaro.org.conf
+++ b/files/apache/snapshots.linaro.org.conf
@@ -17,6 +17,7 @@
 
     # Admin access should be always over SSL
     RewriteEngine On
+    Include /etc/apache2/linaro/block-refs.conf
     RewriteRule ^(/admin.*) https://{{ inventory_hostname }}$1 [redirect=301,noescape,last]
     RewriteRule ^(/login.*) https://{{ inventory_hostname }}$1 [redirect=301,noescape,last]
 
@@ -38,8 +39,6 @@
     SetEnv HOST_NAME "{{ inventory_hostname }}"
     WSGIScriptAlias / /srv/{{ inventory_hostname }}/{{ repo_name }}/wsgi_production.py
 
-    SetEnvIf User-Agent "Amazon Route 53 Health Check" dontlog
-
     <Directory /srv/{{ inventory_hostname }}/{{ repo_name }}>
         <Files wsgi_production.py>
             Require all granted
@@ -71,6 +70,9 @@
         Header set Cache-Control "max-age=86400, public"
     </FilesMatch>
 
+    RewriteEngine On
+    Include /etc/apache2/linaro/block-refs.conf
+
     Include /etc/apache2/linaro/headers-https.conf
     Include /etc/apache2/linaro/settings-ssl.conf
     SSLCertificateFile {{ ssl_cert }}
@@ -93,8 +95,6 @@
     SetEnv HOST_NAME "{{ inventory_hostname }}"
     WSGIScriptAlias / /srv/{{ inventory_hostname }}/{{ repo_name }}/wsgi_production.py
 
-    SetEnvIf User-Agent "Amazon Route 53 Health Check" dontlog
-
     <Directory /srv/{{ inventory_hostname }}/{{ repo_name }}>
         <Files wsgi_production.py>
             Require all granted
diff --git a/files/apache/source.devboardsforandroid.linaro.org.conf b/files/apache/source.devboardsforandroid.linaro.org.conf
index d171d19f..53335e15 100644
--- a/files/apache/source.devboardsforandroid.linaro.org.conf
+++ b/files/apache/source.devboardsforandroid.linaro.org.conf
@@ -3,14 +3,17 @@
 
 	RedirectMatch permanent "^/(?!\.well-known/acme-challenge)(.*)" "https://source.devboardsforandroid.linaro.org/$1"
 
+    RewriteEngine On
+    Include /etc/apache2/linaro/block-refs.conf
 
-        Include /etc/apache2/linaro/letsencrypt.conf
+
+    Include /etc/apache2/linaro/letsencrypt.conf
 </VirtualHost>
 
 <VirtualHost *:443>
 	ServerName source.devboardsforandroid.linaro.org
 
-        Include /etc/apache2/linaro/settings-ssl.conf
+    Include /etc/apache2/linaro/settings-ssl.conf
 	SSLCertificateFile /etc/dehydrated/certs/source.devboardsforandroid.linaro.org/fullchain.pem
 	SSLCertificateKeyFile /etc/dehydrated/certs/source.devboardsforandroid.linaro.org/privkey.pem
 	SSLCACertificateFile /etc/dehydrated/certs/source.devboardsforandroid.linaro.org/fullchain.pem
@@ -61,5 +64,6 @@
 	ProxyPass / http://127.0.0.1:8080/plugins/gitiles/ retry=0 nocanon Keepalive=On
 
 	RewriteEngine On
+    Include /etc/apache2/linaro/block-refs.conf
 	RewriteRule ^/plugins/gitiles(.+)$ https://source.devboardsforandroid.linaro.org$1 [L,R=301,NE]
 </VirtualHost>
diff --git a/files/apache/testdata.lava.morello-project.org.conf b/files/apache/testdata.lava.morello-project.org.conf
index 5b39c023..f160fdeb 100644
--- a/files/apache/testdata.lava.morello-project.org.conf
+++ b/files/apache/testdata.lava.morello-project.org.conf
@@ -3,6 +3,9 @@
     ServerAdmin webmaster@linaro.org
     DocumentRoot /srv/testdata
 
+    RewriteEngine On
+    Include /etc/apache2/linaro/block-refs.conf
+
     <Directory "/srv/testdata">
         Order allow,deny
         Allow from all
diff --git a/files/apache/vhost-obs.conf b/files/apache/vhost-obs.conf
index 5c8b0a43..41598459 100644
--- a/files/apache/vhost-obs.conf
+++ b/files/apache/vhost-obs.conf
@@ -14,6 +14,7 @@
 
    RewriteEngine on
    Include /etc/apache2/linaro/letsencrypt.conf
+   Include /etc/apache2/linaro/block-refs.conf
 
    <Directory "/srv/obs.linaro.org">
      AllowOverride None
@@ -42,6 +43,9 @@
    SSLCertificateKeyFile {{ssl_key}}
    SSLCACertificateFile {{ssl_ca}}
 
+   RewriteEngine On
+   Include /etc/apache2/linaro/block-refs.conf
+
    <Directory "/srv/obs.linaro.org">
      AllowOverride None
      Require all granted
diff --git a/roles/apache-site/files/block-refs.conf b/roles/apache-site/files/block-refs.conf
new file mode 100644
index 00000000..09bc23bb
--- /dev/null
+++ b/roles/apache-site/files/block-refs.conf
@@ -0,0 +1,4 @@
+    SetEnvIf User-Agent "Amazon Route 53 Health Check" dontlog
+
+    RewriteCond %{HTTP_USER_AGENT} (AhrefsBot|amazonbot|bingbot|Baidu|Baiduspider|360Spider|360|MauiBot|SemrushBot|MegaIndex|PetalBot) [nocase]
+    RewriteRule ^(.*)$ - [forbidden,last]