Improve customization for rancher orchestration

- Delegate multiple django settings to container environment
  - SECRET_KEY for Django
  - MAPQUEST_MAP_KEY for MapQuest basemap
  - SITE_DOMAIN_NAME for InaSAFE Realtime
- Bugfix for production specific containers
- Bugfix for development orchestration

.gitignore

@@ -2,7 +2,6 @@
 *.*~
 
 # project specifc
-django_project/core/settings/secret.py
 xmlrunner
 
 # Byte-compiled / optimized / DLL files
@@ -107,9 +106,6 @@ deployment/docker-compose.override.yml
 !deployment/sftp_backup/target/pg/.gitkeep
 !deployment/sftp_backup/backups/media/README.md
 !deployment/sftp_backup/backups/pg/README.md
-django_project/realtime/tasks/realtime/celeryconfig.py
-django_project/core/settings/custom_settings.py
-django_project/core/settings/mapquest.py
 django_project/realtime/local_settings.py
 django_project/celerybeat.pid
 django_project/temp

deployment/ansible/development/roles/docker_compose/templates/docker-compose.override.yml.j2

@@ -34,6 +34,10 @@ uwsgi:
     - VIRTUAL_HOST=users.inasafe.org
     - VIRTUAL_PORT=8080
     - BROKER_URL=amqp://guest:guest@rabbitmq:5672/
+    - INASAFE_REALTIME_BROKER_URL=amqp://guest:guest@{{ interpreters.inasafe_realtime.ipaddress }}:{{ docker_port_forward.inasafe_realtime.rabbitmq }}/
+    # Secrets
+    - MAPQUEST_MAP_KEY={{ inasafe_django.mapquest_map_key }}
+    - SITE_DOMAIN_NAME={{ inasafe_django.site_domain_name }}
   links:
     - smtp:smtp
     - db:db

deployment/ansible/development/roles/pycharm/templates/run-manager-environment.xml.j2

@@ -1,5 +1,12 @@
 
         <env name="BROKER_URL" value="amqp://guest:guest@rabbitmq:5672/" />
         <env name="DJANGO_SETTINGS_MODULE" value="core.settings.dev_docker" />
+        <env name="DATABASE_NAME" value="gis" />
+        <env name="DATABASE_USERNAME" value="docker" />
+        <env name="DATABASE_PASSWORD" value="docker" />
+        <env name="DATABASE_HOST" value="db" />
+        <env name="INASAFE_REALTIME_BROKER_URL" value="amqp://guest:guest@{{ interpreters.inasafe_realtime.ipaddress }}:{{ docker_port_forward.inasafe_realtime.rabbitmq }}/" />
+        <env name="MAPQUEST_MAP_KEY" value="{{ inasafe_django.mapquest_map_key }}" />
+        <env name="SITE_DOMAIN_NAME" value="{{ inasafe_django.site_domain_name }}" />
         <env name="PYTHONUNBUFFERED" value="1" />
         <env name="C_FORCE_ROOT" value="true" />

deployment/ansible/development/site.yml

@@ -5,7 +5,6 @@
   roles:
     - { role: pycharm, when: use_pycharm, tags: development }
     - docker_compose
-    - inasafe_django
   vars_prompt:
     - name: "ansible_sudo_pass"
       prompt: "Sudo password"

deployment/production/docker/db/Dockerfile

@@ -0,0 +1,4 @@
+FROM kartoza/postgis:9.3-2.1
+
+ADD pg_hba.conf /etc/postgresql/9.3/main/pg_hba.conf
+RUN chown postgres:postgres /etc/postgresql/9.3/main/pg_hba.conf

deployment/production/docker/db/build.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+IMAGE_NAME=inasafe-django_db
+TAG_NAME=v3.4
+docker build -t kartoza/${IMAGE_NAME} .
+docker tag kartoza/${IMAGE_NAME}:latest kartoza/${IMAGE_NAME}:${TAG_NAME}

deployment/production/docker/db/pg_hba.conf

@@ -0,0 +1,102 @@
+# PostgreSQL Client Authentication Configuration File
+# ===================================================
+#
+# Refer to the "Client Authentication" section in the PostgreSQL
+# documentation for a complete description of this file.  A short
+# synopsis follows.
+#
+# This file controls: which hosts are allowed to connect, how clients
+# are authenticated, which PostgreSQL user names they can use, which
+# databases they can access.  Records take one of these forms:
+#
+# local      DATABASE  USER  METHOD  [OPTIONS]
+# host       DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
+# hostssl    DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
+# hostnossl  DATABASE  USER  ADDRESS  METHOD  [OPTIONS]
+#
+# (The uppercase items must be replaced by actual values.)
+#
+# The first field is the connection type: "local" is a Unix-domain
+# socket, "host" is either a plain or SSL-encrypted TCP/IP socket,
+# "hostssl" is an SSL-encrypted TCP/IP socket, and "hostnossl" is a
+# plain TCP/IP socket.
+#
+# DATABASE can be "all", "sameuser", "samerole", "replication", a
+# database name, or a comma-separated list thereof. The "all"
+# keyword does not match "replication". Access to replication
+# must be enabled in a separate record (see example below).
+#
+# USER can be "all", a user name, a group name prefixed with "+", or a
+# comma-separated list thereof.  In both the DATABASE and USER fields
+# you can also write a file name prefixed with "@" to include names
+# from a separate file.
+#
+# ADDRESS specifies the set of hosts the record matches.  It can be a
+# host name, or it is made up of an IP address and a CIDR mask that is
+# an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that
+# specifies the number of significant bits in the mask.  A host name
+# that starts with a dot (.) matches a suffix of the actual host name.
+# Alternatively, you can write an IP address and netmask in separate
+# columns to specify the set of hosts.  Instead of a CIDR-address, you
+# can write "samehost" to match any of the server's own IP addresses,
+# or "samenet" to match any address in any subnet that the server is
+# directly connected to.
+#
+# METHOD can be "trust", "reject", "md5", "password", "gss", "sspi",
+# "krb5", "ident", "peer", "pam", "ldap", "radius" or "cert".  Note that
+# "password" sends passwords in clear text; "md5" is preferred since
+# it sends encrypted passwords.
+#
+# OPTIONS are a set of options for the authentication in the format
+# NAME=VALUE.  The available options depend on the different
+# authentication methods -- refer to the "Client Authentication"
+# section in the documentation for a list of which options are
+# available for which authentication methods.
+#
+# Database and user names containing spaces, commas, quotes and other
+# special characters must be quoted.  Quoting one of the keywords
+# "all", "sameuser", "samerole" or "replication" makes the name lose
+# its special character, and just match a database or username with
+# that name.
+#
+# This file is read on server startup and when the postmaster receives
+# a SIGHUP signal.  If you edit the file on a running system, you have
+# to SIGHUP the postmaster for the changes to take effect.  You can
+# use "pg_ctl reload" to do that.
+
+# Put your actual configuration here
+# ----------------------------------
+#
+# If you want to allow non-local connections, you need to add more
+# "host" records.  In that case you will also need to make PostgreSQL
+# listen on a non-local interface via the listen_addresses
+# configuration parameter, or via the -i or -h command line switches.
+
+
+
+
+# DO NOT DISABLE!
+# If you change this first entry you will need to make sure that the
+# database superuser can access the database using some other method.
+# Noninteractive access to all databases is required during automatic
+# maintenance (custom daily cronjobs, replication, and similar tasks).
+#
+# Database administrative login by Unix domain socket
+local   all             postgres                                peer
+
+# TYPE  DATABASE        USER            ADDRESS                 METHOD
+
+# "local" is for Unix domain socket connections only
+local   all             all                                     peer
+# IPv4 local connections:
+host    all             all             127.0.0.1/32            md5
+# IPv6 local connections:
+host    all             all             ::1/128                 md5
+# Allow replication connections from localhost, by a user with the
+# replication privilege.
+#local   replication     postgres                                peer
+#host    replication     postgres        127.0.0.1/32            md5
+#host    replication     postgres        ::1/128                 md5
+
+# Allow connection from arbitrary docker containers in the docker network
+host	all				all				0.0.0.0/0				md5

deployment/production/docker/docker-compose.yml

@@ -1,12 +1,33 @@
 version: '2'
 volumes:
   django-postgis-data:
+    driver: rancher-nfs
+    driver_opts:
+      onRemove: retain
   django-media-data:
+    driver: rancher-nfs
+    driver_opts:
+      onRemove: retain
   django-static-data:
+    driver: rancher-nfs
+    driver_opts:
+      onRemove: retain
   django-postgis-dbbackup-data:
+    driver: rancher-nfs
+    driver_opts:
+      onRemove: retain
   sftpbackup-postgis-target-data:
+    driver: rancher-nfs
+    driver_opts:
+      onRemove: retain
   sftpbackup-postgis-history-data:
+    driver: rancher-nfs
+    driver_opts:
+      onRemove: retain
   sftpbackup-media-history-data:
+    driver: rancher-nfs
+    driver_opts:
+      onRemove: retain
 services:
   smtp:
     image: catatnight/postfix
@@ -16,7 +37,7 @@ services:
       - smtp_user=noreply:docker
 
   db:
-    image: kartoza/postgis:9.3-2.1
+    image: kartoza/inasafe-django_db:v3.4
     restart: on-failure
     volumes:
       - django-postgis-data:/var/lib/postgresql
@@ -36,7 +57,7 @@ services:
       - db:db
 
   dbbackup: &dbbackup
-    image: inasafe-django_dbbackup
+    image: kartoza/inasafe-django_dbbackup:v3.4
     volumes:
       - django-postgis-dbbackup-data:/backups
       # this folder is used as target folder for sftppgbackup as the
@@ -76,7 +97,6 @@ services:
       - SFTP_USER=user
       - SFTP_PASSWORD=password
       - SFTP_DIR=/
-      - TARGET_FOLDER=/pg_backup
 
 
   sftpmediabackup: &sftpmediabackup
@@ -101,10 +121,9 @@ services:
       - SFTP_USER=user
       - SFTP_PASSWORD=password
       - SFTP_DIR=/
-      - TARGET_FOLDER=/pg_backup
 
   uwsgi: &uwsgi
-    image: inasafe-django_uwsgi
+    image: kartoza/inasafe-django_uwsgi:v3.4
     restart: on-failure
     environment:
       - DATABASE_NAME=gis
@@ -115,6 +134,10 @@ services:
       - VIRTUAL_HOST=realtime.inasafe.org
       - VIRTUAL_PORT=8080
       - BROKER_URL=amqp://guest:guest@rabbitmq:5672/
+      # Secrets
+      - SECRET_KEY=
+      - MAPQUEST_MAP_KEY=
+      - SITE_DOMAIN_NAME=
       # InAWARE Credentials
       - INAWARE_HOST=
       - INAWARE_USER=
@@ -155,7 +178,7 @@ services:
     command: celery -A core.celery_app worker --loglevel=info -Q inasafe-django-indicator -B -n indicator.%h
 
   web: &web
-    image: inasafe-django_nginx
+    image: kartoza/inasafe-django_nginx:v3.4
     volumes:
       - django-static-data:/home/web/static:ro
       - django-media-data:/home/web/media:ro

deployment/production/docker/uwsgi/Dockerfile

@@ -16,16 +16,16 @@ RUN pip install uwsgi
 RUN apt-get update -y; apt-get install -y gettext
 
 # Clone source code
-
-ADD entry-point.sh /entry-point.sh
-
 RUN mkdir -p /home/src/
 RUN mkdir -p /home/web/
-RUN cd /home/src
+WORKDIR /home/src
 RUN git clone --branch v3.4 --depth 1 https://github.com/inasafe/inasafe-django.git
 
 # Copy only django_project folder
-RUN cp -r /home/src/django_project /home/web/
+RUN mkdir -p /home/web
+RUN cp -r inasafe-django/django_project /home/web/
 
 WORKDIR /home/web/django_project
+
+ADD entry-point.sh /entry-point.sh
 CMD ["/entry-point.sh"]

deployment/production/docker/uwsgi/build.sh

@@ -1,2 +1,5 @@
 #!/bin/sh
- docker build -t inasafe/inasafe-django_uwsgi .
+IMAGE_NAME=inasafe-django_uwsgi
+TAG_NAME=v3.4
+docker build -t kartoza/${IMAGE_NAME} .
+docker tag kartoza/${IMAGE_NAME}:latest kartoza/${IMAGE_NAME}:${TAG_NAME}

deployment/production/docker/uwsgi/entry-point.sh

@@ -1,10 +1,12 @@
 #!/usr/bin/env bash
 
 # Run database migrations
+echo "Run database migrations"
 ./manage.py migrate
 
 # Run collectstatic
-./manage.py collectstatic
+echo "Run collectstatic"
+./manage.py collectstatic --noinput
 
 # Run uwsgi
-uwsgi --ini uwsgi.conf
+uwsgi --ini /uwsgi.conf

django_project/core/settings/base.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 # Django settings for inasafe project.
+import os
 import sys
 
 from .utils import ABS_PATH
@@ -53,7 +54,7 @@
 )
 
 # import SECRET_KEY into current namespace
-from .secret import SECRET_KEY  # noqa
+from .secret import *  # noqa
 
 # default middleware classes
 MIDDLEWARE_CLASSES = (
@@ -105,4 +106,4 @@
 
 DJANGO_ROOT = ABS_PATH('.')
 
-SITE_DOMAIN_NAME = 'example.com'
+SITE_DOMAIN_NAME = os.environ.get('SITE_DOMAIN_NAME', 'example.com')

django_project/core/settings/contrib.py

@@ -69,8 +69,4 @@
     'file_browser_callback': 'mce_filebrowser'
 }
 
-try:
-    # This settings will be used for mapquest tiles for realtime and user_map
-    from .mapquest import MAPQUEST_MAP_KEY  # noqa
-except:
-    pass
+from .secret import MAPQUEST_MAP_KEY  # noqa