Publishing: Build our publishing server on docker containers

Change-Id: I1daf77b5642732958aa59011b773a460df6ca7ad
Reviewed-on: https://review.linaro.org/21174
Reviewed-by: Ben Copeland <ben.copeland@linaro.org>

8 files changed, 171 insertions, 108 deletions

diff --git a/files/publishing/allowed_hosts.txt b/files/publishing/allowed_hosts.txt
new file mode 100644
index 00000000..bdef4181
--- /dev/null
+++ b/files/publishing/allowed_hosts.txt
@@ -0,0 +1,8 @@
+localhost
+127.0.0.1
+releases.linaro.org
+snapshots.linaro.org
+builds.linaro.org
+{% for item in docker_containers %}
+{{item.host}}
+{% endfor %}
diff --git a/files/publishing/llp-apache.conf b/files/publishing/llp-apache.conf
index eede6086..9103143c 100644
--- a/files/publishing/llp-apache.conf
+++ b/files/publishing/llp-apache.conf
@@ -1,124 +1,33 @@
 <VirtualHost *:80>
-    Alias /static/ {{django_app_root}}/static/
-    DocumentRoot /srv/{{ hostname }}/www
-    ServerName {{ publishing_host }}
-    ServerAlias {{ hostname }}
-    ServerAdmin webmaster@linaro.org
-    CustomLog /var/log/apache2/{{ hostname }}-access.log combined env=!dontlog
-    ErrorLog  /var/log/apache2/{{ hostname }}-error.log
+    ServerName {{config}}
+    ServerAlias {{config}}
+    ServerAdmin {{email}}
+    CustomLog /var/log/apache2/{{config}}-access.log combined env=!dontlog
+    ErrorLog  /var/log/apache2/{{config}}-error.log
 
-    # Admin access should be always over SSL
-    RewriteEngine On
-    RewriteRule ^(/admin.*) https://{{hostname}}$1 [redirect=301,noescape,last]
-
-    <Directory {{django_app_root}}/static>
-        <IfVersion < 2.3 >
-            Order allow,deny
-            Allow from all
-        </IfVersion>
-        <IfVersion >= 2.3>
-            Require all granted
-        </IfVersion>
-    </Directory>
-
-    <Directory /srv/{{ hostname }}/www>
-        Order deny,allow
-        Allow from all
-    </Directory>
-
-    XSendFile on
-    XSendFilePath /srv/{{ hostname }}/www
-
-    SetEnv SITE_NAME "{{ sitename }}"
-    SetEnv HOST_NAME "{{ hostname }}"
-    WSGIScriptAlias / {{django_app_root}}/wsgi_production.py
+    ProxyPass / http://localhost:808{{PORT}}/
+    ProxyPassReverse / http://localhost:808{{PORT}}/
 
     SetEnvIf User-Agent "Amazon Route 53 Health Check" dontlog
 
-    <Directory {{django_app_root}}>
-        <Files wsgi_production.py>
-            <IfVersion < 2.3 >
-                Order allow,deny
-                Allow from all
-            </IfVersion>
-            <IfVersion >= 2.3>
-                Require all granted
-            </IfVersion>
-        </Files>
-    </Directory>
-
-    <Directory />
-        IndexOptions NameWidth=* +SuppressDescription
-        Options +Indexes +FollowSymLinks
-        IndexIgnore favicon.ico HEADER.html
-        AllowOverride FileInfo Indexes
-    </Directory>
 
 </VirtualHost>
 
 <VirtualHost *:443>
-    Alias /static/ {{django_app_root}}/static/
-    DocumentRoot /srv/{{ hostname }}/www
-    ServerName {{ publishing_host }}
-    ServerAlias {{ hostname }}
-    ServerAdmin webmaster@linaro.org
-    CustomLog /var/log/apache2/{{ hostname }}-access.log combined env=!dontlog
-    ErrorLog  /var/log/apache2/{{ hostname }}-error.log
+    ServerName {{config}}
+    ServerAlias {{config}}
+    ServerAdmin {{email}}
+    CustomLog /var/log/apache2/{{config}}-access.log combined env=!dontlog
+    ErrorLog  /var/log/apache2/{{config}}-error.log
 
-    SSLEngine On
-    SSLProtocol All -SSLv2 -SSLv3
-    SSLCompression Off
-    SSLHonorCipherOrder On
-    SSLOptions +StdEnvVars
-    SSLCipherSuite "EDH+CAMELLIA:EDH+aRSA:EECDH+aRSA+AESGCM:EECDH+aRSA+SHA384:\
-    EECDH+aRSA+SHA256:EECDH:+CAMELLIA256:+AES256:+CAMELLIA128:+AES128:+SSLv3:\
-    !aNULL:!eNULL:!LOW:!3DES:!MD5:!EXP:!PSK:!DSS:!RC4:!SEED:!ECDSA:\
-    CAMELLIA256-SHA:AES256-SHA:CAMELLIA128-SHA:AES128-SHA"
-    SSLCertificateFile {{ssl_cert}}
-    SSLCertificateKeyFile {{ssl_key}}
-    SSLCACertificateFile {{ssl_ca_cert}}
+    SSLEngine on
+    SSLCertificateFile /etc/letsencrypt/live/{{config}}/fullchain.pem
+    SSLCertificateKeyFile /etc/letsencrypt/live/{{config}}/privkey.pem
 
-    <Directory {{django_app_root}}/static>
-        <IfVersion < 2.3 >
-            Order allow,deny
-            Allow from all
-        </IfVersion>
-        <IfVersion >= 2.3>
-            Require all granted
-        </IfVersion>
-    </Directory>
-
-    <Directory /srv/{{ hostname }}/www>
-        Order deny,allow
-        Allow from all
-    </Directory>
-
-    XSendFile on
-    XSendFilePath /srv/{{ hostname }}/www
-
-    SetEnv SITE_NAME "{{ sitename }}"
-    SetEnv HOST_NAME "{{ hostname }}"
-    WSGIScriptAlias / {{django_app_root}}/wsgi_production.py
+    ProxyPass / http://localhost:808{{PORT}}/
+    ProxyPassReverse / http://localhost:808{{PORT}}/
 
     SetEnvIf User-Agent "Amazon Route 53 Health Check" dontlog
 
-    <Directory {{django_app_root}}>
-        <Files wsgi_production.py>
-            <IfVersion < 2.3 >
-                Order allow,deny
-                Allow from all
-            </IfVersion>
-            <IfVersion >= 2.3>
-                Require all granted
-            </IfVersion>
-        </Files>
-    </Directory>
-
-    <Directory />
-        IndexOptions NameWidth=* +SuppressDescription
-        Options +Indexes +FollowSymLinks
-        IndexIgnore favicon.ico HEADER.html
-        AllowOverride FileInfo Indexes
-    </Directory>
 
 </VirtualHost>
diff --git a/group_vars/publishing-docker b/group_vars/publishing-docker
new file mode 100644
index 00000000..ee7adb50
--- /dev/null
+++ b/group_vars/publishing-docker
@@ -0,0 +1,33 @@
+# Apache
+hostname: "{{inventory_hostname}}"
+apache_user: www-data
+apache_modules:
+  - ssl
+  - proxy
+  - proxy_http
+  - rewrite
+
+# Docker
+docker_image: 'linaro/ci-amd64-llp-alpine'
+docker_version: 'latest'
+
+docker_http_port: '8080'
+
+django_manage_py: /srv/linaro-license-protection/manage.py
+
+# LDAP
+ldap_cache_url: ""
+
+login_groups:
+  - systems
+  - team-builds-and-baselines
+
+ldap_base: dc=linaro,dc=org
+ldap_uri: "ldaps://login.linaro.org"
+ldap_binddn: cn=ldapbind,dc=linaro,dc=org
+
+# SUDO
+sudo_groups:
+  - systems
+  - users
+
diff --git a/host_vars/publishing-ap.linaro.org b/host_vars/publishing-ap.linaro.org
new file mode 100644
index 00000000..59e5a8c8
--- /dev/null
+++ b/host_vars/publishing-ap.linaro.org
@@ -0,0 +1,5 @@
+docker_containers:
+  - host: builds-ap.96boards.org
+    header_override: 96boards_header.html
+  - host: snapshots-ap.linaro.org
+  - host: releases-ap.linaro.org
diff --git a/hosts b/hosts
index 646c3652..0b74809a 100644
--- a/hosts
+++ b/hosts
@@ -68,3 +68,6 @@ ci.linaro.org
 88.99.28.12     # x86_64-11 (RPB)
 88.99.28.38     # x86_64-12 (RPB)
 88.99.59.232    # x86_64-13 (LITE)
+
+[publishing-docker]
+publishing-ap.linaro.org
diff --git a/publishing-docker.yml b/publishing-docker.yml
new file mode 100644
index 00000000..7afaaae1
--- /dev/null
+++ b/publishing-docker.yml
@@ -0,0 +1,9 @@
+---
+- name: Deploy publishing server
+  hosts: publishing-docker
+  become: yes
+  vars_files:
+    - "{{secrets_dir}}/host_vars/{{inventory_hostname}}"
+  roles:
+    - {role: ssh-ldap, tags: [ssh-ldap]}
+    - {role: publishing-docker-deps, tags: [docker]} # Setups Apache conf and letsencrypt
diff --git a/roles/publishing-docker-deps/tasks/main.yml b/roles/publishing-docker-deps/tasks/main.yml
new file mode 100644
index 00000000..b95806b7
--- /dev/null
+++ b/roles/publishing-docker-deps/tasks/main.yml
@@ -0,0 +1,84 @@
+---
+
+- name: Add Docker apt key
+  apt_key:
+    id: "58118E89F3A912897C070ADBF76221572C52609D"
+    keyserver: "hkp://p80.pool.sks-keyservers.net"
+
+# https://launchpad.net/~canonical-kernel-team/+archive/ubuntu/ppa
+- name: Add repos
+  apt_repository:
+    repo: "{{item}}"
+  with_items:
+    - 'deb https://apt.dockerproject.org/repo ubuntu-xenial main'
+    - 'ppa:canonical-kernel-team/ppa'
+
+- name: Install server deps
+  apt: pkg={{item}} state=installed
+  with_items:
+    - apache2
+    - docker-engine
+    - git
+      #    - linux-image-4.10.0-23-generic
+    - python-pip
+
+- name: Install docker python
+  pip: name=docker-py version=1.10.6
+
+- name: Enable Apache modules
+  apache2_module: name={{item}}
+  with_items: "{{apache_modules}}"
+
+- name: Stop docker container
+  docker_container:
+    name: "{{item.host}}"
+    state: stopped
+  when: docker_upgrade is defined
+  with_items: "{{docker_containers}}"
+
+- name: Create docker container
+  docker_container:
+    name: "{{item.1.host}}"
+    image: "{{docker_image}}"
+    pull: true
+    state: started
+    log_driver: journald
+    restart_policy: unless-stopped
+    recreate: yes
+    published_ports:
+        - "808{{item.0}}:808{{item.0}}"
+    env:
+        PORT: "808{{item.0}}"
+        DJANGO_MIGRATE: "yes"
+        DJANGO_COLLECTSTATIC: "yes"
+        secrets_file: "{{lookup('file', '{{secrets_dir}}/publishing-servers/{{item.1.host}}')}}"
+        allowed_hosts: "{{lookup('file', 'files/publishing/allowed_hosts.txt')}}"
+        html_header: "{{ '' if item.1.header_override is not defined else lookup('file', 'files/publishing/' ~ item.1.header_override) }}"
+  with_indexed_items: "{{docker_containers}}"
+
+- name: Ensure crontab is set
+  template:
+    src: llp-crontab.cron
+    dest: /etc/cron.d/llp-crontab
+    owner: root
+    group: root
+    mode: 0644
+
+- name: Ensure SSL (letsencrypt) is setup
+  with_indexed_items: "{{docker_containers}}"
+  include_role:
+      name: letsencrypt
+  vars:
+      hostname: "{{item.1.host}}"
+  tags: letsencrypt, ssl
+
+- name: Apache config
+  with_indexed_items: "{{docker_containers}}"
+  include_role:
+      name: apache-site
+  vars:
+      src: publishing/llp-apache.conf
+      config: "{{item.1.host}}"
+      PORT: "{{item.0}}"
+  tags: apache
+
diff --git a/roles/publishing-docker-deps/templates/llp-crontab.cron b/roles/publishing-docker-deps/templates/llp-crontab.cron
new file mode 100644
index 00000000..987f549a
--- /dev/null
+++ b/roles/publishing-docker-deps/templates/llp-crontab.cron
@@ -0,0 +1,12 @@
+# DO NOT EDIT: MANAGED BY ANSIBLE
+{% for host in docker_container_name %}
+{% if host.report_process is defined %}
+*/10 * * * * {{apache_user}} docker exec {{host.host}} {{django_manage_py}} report_process --settings=settings_production
+{% endif %}
+{% if host.s3_sanity is defined %}
+0 0 * * 0 {{apache_user}} docker exec {{host.host}} {{django_manage_py}} s3_sanity --settings=settings_production
+{% endif %}
+{% if host.s3_purge is defined %}
+0 0 * * 0 {{apache_user}} docker exec {{host.host}} {{django_manage_py}} s3_purge --settings=settings_production {{host.s3_purge_options|default('')}}
+{% endif %}
+{% endfor %}