From f3f4c52608ed47422b0dad2a5e35652d8363fa39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20K=C3=B6gl?= Date: Wed, 19 Nov 2014 22:23:08 +0100 Subject: [PATCH 01/30] Start Docker support [WIP] --- .dockerignore | 12 +++++ Dockerfile | 41 ++++++++++++++ bin/docker-env.sh | 17 ++++++ contrib/init-db.sql | 9 ++++ doc/dev/postgres-setup.rst | 14 ++--- doc/index.rst | 1 + doc/ops/configuration.rst | 106 +++++++++++++++++++++++++++++++++++++ doc/ops/docker.rst | 61 +++++++++++++++++++++ doc/ops/index.rst | 9 ++++ fig.yml | 2 + makefile | 5 ++ mygpo/settings.py | 51 ++++++++++++++---- 12 files changed, 306 insertions(+), 22 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100755 bin/docker-env.sh create mode 100644 contrib/init-db.sql create mode 100644 doc/ops/configuration.rst create mode 100644 doc/ops/docker.rst create mode 100644 doc/ops/index.rst create mode 100644 fig.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..fa795eb6c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +.git +htdocs/media/logo +mygpo/settings_prod.py +doc/ +res/ +tools/ +locale/ +venv* +*.pyc +*/*.pyc +*/*/*.pyc +*/*/*/*.pyc diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..720c4f268 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,41 @@ +FROM ubuntu:14.04 +MAINTAINER Stefan Kögl + +# use bash instead of built-in sh, because it does not support "source" during build +RUN rm /bin/sh && ln -s /bin/bash /bin/sh + +# install all packaged dependencies +RUN apt-get update && apt-get install -y \ + git \ + python2.7 \ + python2.7-dev \ + python-virtualenv \ + libpq-dev \ + libjpeg-dev \ + zlib1g-dev \ + libwebp-dev + +# create log directories +RUN mkdir -p /var/log/gunicorn + +# copy source +COPY . /srv/mygpo +WORKDIR /srv/mygpo + +# run everything in a virtualenv +RUN virtualenv venv +RUN source venv/bin/activate + +# install all runtime dependencies +RUN pip install \ + -r /srv/mygpo/requirements.txt \ + -r /srv/mygpo/requirements-setup.txt + +# set up missing environment variables +ENTRYPOINT ["/srv/mygpo/bin/docker-env.sh"] + +# default launch command +CMD ["gunicorn", "mygpo.wsgi:application", "--access-logfile", "-"] + +# HTTP port +EXPOSE 8000 diff --git a/bin/docker-env.sh b/bin/docker-env.sh new file mode 100755 index 000000000..3d7ca6aef --- /dev/null +++ b/bin/docker-env.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Docker doesn't have a great way to set environment variables at startup. +# This scripts will set up some defaults. + +# if a DATABSE_URL is provided from outside, use it +if [[ -z "$DATABASE_URL" ]]; then + # otherwise construct one using a linked "db" container + export DATABASE_URL="postgres://mygpo:mygpo@${DB_PORT_5432_TCP_ADDR}:5432/mygpo" +fi + +# if not SECRET_KEY is provided from outside, create a random one +if [[ -z "$SECRET_KEY" ]]; then + export SECRET_KEY=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1) +fi + +# Execute the commands passed to this script +exec "$@" diff --git a/contrib/init-db.sql b/contrib/init-db.sql new file mode 100644 index 000000000..5b4dad338 --- /dev/null +++ b/contrib/init-db.sql @@ -0,0 +1,9 @@ +CREATE USER mygpo WITH PASSWORD 'mygpo'; +ALTER USER mygpo CREATEDB; -- required for creating test database +CREATE DATABASE mygpo; +CREATE DATABASE test_mygpo; +GRANT ALL PRIVILEGES ON DATABASE mygpo to mygpo; +GRANT ALL PRIVILEGES ON DATABASE test_mygpo to mygpo; +ALTER DATABASE mygpo OWNER TO mygpo; +ALTER DATABASE test_mygpo OWNER TO mygpo; +ALTER ROLE mygpo SET statement_timeout = 5000; diff --git a/doc/dev/postgres-setup.rst b/doc/dev/postgres-setup.rst index dea968957..1f7929d9d 100644 --- a/doc/dev/postgres-setup.rst +++ b/doc/dev/postgres-setup.rst @@ -3,14 +3,6 @@ PostgreSQL Setup Use the following to set up a local PostgreSQL. -.. code-block:: sql - - CREATE USER mygpo WITH PASSWORD 'mygpo'; - ALTER USER mygpo CREATEDB; -- required for creating test database - CREATE DATABASE mygpo; - CREATE DATABASE test_mygpo; - GRANT ALL PRIVILEGES ON DATABASE mygpo to mygpo; - GRANT ALL PRIVILEGES ON DATABASE test_mygpo to mygpo; - ALTER DATABASE mygpo OWNER TO mygpo; - ALTER DATABASE test_mygpo OWNER TO mygpo; - ALTER ROLE mygpo SET statement_timeout = 5000; +.. literalinclude:: ../../contrib/init-db.sql + :language: sql + :linenos: diff --git a/doc/index.rst b/doc/index.rst index 2bc6fa7e1..be1f76bef 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -24,6 +24,7 @@ Contents publisher/index api/index dev/index + ops/index Translator Documentation diff --git a/doc/ops/configuration.rst b/doc/ops/configuration.rst new file mode 100644 index 000000000..9e4e32848 --- /dev/null +++ b/doc/ops/configuration.rst @@ -0,0 +1,106 @@ +Configuration +============= + +Configuration can be done through the following environment variables. + +``DEBUG`` +--------- +Debug mode shows error pages, enables debug output, etc. + + +``DATABASE_URL`` +---------------- +DB connection string in the form of ``postgres://USER:PASSWORD@HOST:PORT/NAME`` + + +``ACCOUNT_ACTIVATION_DAYS`` +--------------------------- +Number of days that newly registered users have time to activate their account. + + +``DEFAULT_FROM_EMAIL`` +---------------------- +Default sender address for outgoing emails. See `Django documentation +`__. + + +``SECRET_KEY`` +-------------- +See `Django documentation +`__. + + +``GOOGLE_ANALYTICS_PROPERTY_ID`` +-------------------------------- +Setting a Google Analytics Property ID activates GA integration. + + +``DIRECTORY_EXCLUDED_TAGS`` +--------------------------- +A comma-separated list of tags that are excluded from the directory. + + +``FLICKR_API_KEY`` +------------------ +Setting a Flickr API key activates Flickr integration. + + +``MAINTENANCE`` +--------------- +Switches the site into a maintenance mode. + + +* ``PODCAST_SLUG_SUBSCRIBER_LIMIT`` + +* ``MIN_SUBSCRIBERS_CATEGORY``: minimum number of subscribers that a podcast + needs to "push" one of its categories to the top + +* ``API_ACTIONS_MAX_NONBG``: maximum number of episode actions that the API + processes immediatelly before returning the response. Larger requests will + be handled in background. + +* ``ADSENSE_CLIENT`` + +* ``ADSENSE_SLOT_BOTTOM`` + +* ``STAFF_TOKEN``: enabled access to staff-only areas with ?staff= + +* ``FLATTR_KEY`` + +* ``FLATTR_SECRET`` + +* ``FLATTR_MYGPO_THING``: Flattr thing of the webservice. Will be flattr'd + when a user sets the "Auto-Flattr gpodder.net" option + +* ``USER_AGENT``: The User-Agent string used for outgoing HTTP requests + +* ``DEFAULT_BASE_URL``: Base URL of the website that is used if the actually + used parameters is not available. Request handlers, for example, can access + the requested domain. Code that runs in background can not do this, and + therefore requires a default value. This should be set to something like + ``http://example.com`` + +* ``BROKER_URL`` Celery Broker URL + +* ``CELERY_RESULT_BACKEND`` + +* ``CELERY_SEND_TASK_ERROR_EMAILS`` + +* ``SERVER_EMAIL`` + +* ``GOOGLE_CLIENT_ID`` + +* ``GOOGLE_CLIENT_SECRET`` + +* ``SUPPORT_URL``: URL where users of the site can get support + +* ``ELASTICSEARCH_SERVER`` + +* ``ELASTICSEARCH_INDEX`` + +* ``ELASTICSEARCH_TIMEOUT`` + +* ``ACTIVATION_VALID_DAYS`` time for how long an activation is valid; after + that, an unactivated user will be deleted + +* ``INTERNAL_IPS`` diff --git a/doc/ops/docker.rst b/doc/ops/docker.rst new file mode 100644 index 000000000..9ee0907b1 --- /dev/null +++ b/doc/ops/docker.rst @@ -0,0 +1,61 @@ +Using Docker +============ + +mygpo can be run using `Docker `_. + + +Database +-------- + +The image requires a PostgreSQL server, specified either via + +* a `linked container `_ + called ``db`` containing a server with a database called ``mygpo``, a user + called ``mygpo`` with a password ``mygpo``. +* A ``DATABASE_URL`` environment variable (eg + ``postgres://USER:PASSWORD@HOST:PORT/NAME``) + +Using a PostgreSQL Docker container +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Start a container using the `official PostgreSQL image `_. :: + + docker run --name db -d postgres + +Create the schema and a corresponding user :: + + docker exec -it db psql -U postgres + +And enter the following commands (change passwords as required) + +.. literalinclude:: ../../contrib/init-db.sql + :language: sql + :linenos: + +Initialize the tables. This needs needs to be run for every update. :: + + sudo docker run --rm --link db:db -e SECRET_KEY=asdf mygpo/web python manage.py migrate + + +Elasticsearch +------------- + + +Redis +----- + + +Web Frontend +------------ + +The image exposes the web interface on port 8000. + + +Celery Worker +------------- + + +Celery Heartbeat +---------------- + + diff --git a/doc/ops/index.rst b/doc/ops/index.rst new file mode 100644 index 000000000..4b8d278fe --- /dev/null +++ b/doc/ops/index.rst @@ -0,0 +1,9 @@ +Operation Documentation +======================= + + +.. toctree:: + :maxdepth: 1 + + configuration + docker diff --git a/fig.yml b/fig.yml new file mode 100644 index 000000000..d13cdd25e --- /dev/null +++ b/fig.yml @@ -0,0 +1,2 @@ +db: + image: postgres diff --git a/makefile b/makefile index 13c212490..5f11efdd6 100644 --- a/makefile +++ b/makefile @@ -20,6 +20,11 @@ coverage: clean: find -name "*.pyc" -exec rm '{}' \; +docker-build: + sudo docker build -t="mygpo/web" . + +docker-run: + sudo docker run --rm -p 8000:8000 --name web --link db:db -e SECRET_KEY=asdf mygpo/web .PHONY: all help test clean unittest coverage diff --git a/mygpo/settings.py b/mygpo/settings.py index 9bacce642..aa42797db 100644 --- a/mygpo/settings.py +++ b/mygpo/settings.py @@ -20,12 +20,29 @@ import os.path import dj_database_url + +def parse_bool(s): + """ parses a boolean setting """ + if isinstance(s, bool): + return s + s = s.lower.strip() + return s not in ('n', 'no', 'false', '0', 'off') + + +def parse_int(s): + return int(str(s)) + + +def parse_strlist(s): + return [item.strip() for item in s.split(',')] + + BASE_DIR = os.path.dirname(os.path.abspath(__file__)) # http://code.djangoproject.com/wiki/BackwardsIncompatibleChanges#ChangedthewayURLpathsaredetermined -FORCE_SCRIPT_NAME="" +FORCE_SCRIPT_NAME = "" -DEBUG = True +DEBUG = parse_bool(os.getenv('DEBUG', True)) TEMPLATE_DEBUG = DEBUG ADMINS = () @@ -146,9 +163,12 @@ 'couchdbkit', ) -TEST_RUNNER='mygpo.test.MygpoTestSuiteRunner' -ACCOUNT_ACTIVATION_DAYS = 7 +TEST_RUNNER = 'mygpo.test.MygpoTestSuiteRunner' + + +ACCOUNT_ACTIVATION_DAYS = parse_int(os.getenv('ACCOUNT_ACTIVATION_DAYS', 7)) + AUTHENTICATION_BACKENDS = ( 'django.contrib.auth.backends.ModelBackend', @@ -184,17 +204,26 @@ LOGIN_URL = '/login/' -CSRF_FAILURE_VIEW='mygpo.web.views.security.csrf_failure' +CSRF_FAILURE_VIEW = 'mygpo.web.views.security.csrf_failure' + + +DEFAULT_FROM_EMAIL = os.getenv('DEFAULT_FROM_EMAIL', '') -# The following entries should be set in settings_prod.py -DEFAULT_FROM_EMAIL = '' SECRET_KEY = os.getenv('SECRET_KEY', '') -GOOGLE_ANALYTICS_PROPERTY_ID='' -DIRECTORY_EXCLUDED_TAGS = () -FLICKR_API_KEY = '' -MAINTENANCE = os.path.exists(os.path.join(BASE_DIR, 'MAINTENANCE')) + +GOOGLE_ANALYTICS_PROPERTY_ID = os.getenv('GOOGLE_ANALYTICS_PROPERTY_ID', '') + + +DIRECTORY_EXCLUDED_TAGS = parse_strlist(os.getenv('DIRECTORY_EXCLUDED_TAGS', + '')) + + +FLICKR_API_KEY = os.getenv('FLICKR_API_KEY', '') + + +MAINTENANCE = parse_bool(os.getenv('MAINTENANCE', False)) LOGGING = { From 3a719f098fdfdaf4c9d2bf412dfdb88f63a8dac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20K=C3=B6gl?= Date: Sun, 2 Jul 2017 15:45:38 +0200 Subject: [PATCH 02/30] Move dependency installation from Dockerfile to makefile --- Dockerfile | 17 +++++++---------- makefile | 2 +- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 720c4f268..d5fd8d673 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,16 +4,13 @@ MAINTAINER Stefan Kögl # use bash instead of built-in sh, because it does not support "source" during build RUN rm /bin/sh && ln -s /bin/bash /bin/sh -# install all packaged dependencies -RUN apt-get update && apt-get install -y \ - git \ - python2.7 \ - python2.7-dev \ - python-virtualenv \ - libpq-dev \ - libjpeg-dev \ - zlib1g-dev \ - libwebp-dev +RUN apt-get update + +# install Docker dependencies +RUN apt-get install -y git virtualenv + +# install all packaged runtime dependencies +RUN yes | make install-deps # create log directories RUN mkdir -p /var/log/gunicorn diff --git a/makefile b/makefile index 65f2e9136..6a18cd896 100644 --- a/makefile +++ b/makefile @@ -13,7 +13,7 @@ clean: install-deps: sudo apt-get install libpq-dev libjpeg-dev zlib1g-dev libwebp-dev \ - build-essential python3-dev virtualenv + build-essential python3-dev docker-build: sudo docker build -t="mygpo/web" . From 9ce0fa2fd4e06dd513c31cf8e0d544bec6034b7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20K=C3=B6gl?= Date: Sun, 2 Jul 2017 16:05:40 +0200 Subject: [PATCH 03/30] Simplify settings --- mygpo/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mygpo/settings.py b/mygpo/settings.py index 05f2fcbb7..dda11f3ea 100644 --- a/mygpo/settings.py +++ b/mygpo/settings.py @@ -44,7 +44,7 @@ def get_intOrNone(name, default): BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -DEBUG = get_bool(os.getenv('DEBUG', True)) +DEBUG = get_bool('DEBUG', False) ADMINS = re.findall(r'\s*([^<]+) <([^>]+)>\s*', os.getenv('ADMINS', '')) @@ -241,7 +241,7 @@ def get_intOrNone(name, default): SOUNDCLOUD_CONSUMER_KEY = os.getenv('SOUNDCLOUD_CONSUMER_KEY', '') -MAINTENANCE = get_bool(os.getenv('MAINTENANCE', False)) +MAINTENANCE = get_bool('MAINTENANCE', False) From 247b23eb15f884f9615633f6203dccbf48810682 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20K=C3=B6gl?= Date: Wed, 19 Nov 2014 22:23:08 +0100 Subject: [PATCH 04/30] Start Docker support [WIP] --- Dockerfile | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index d5fd8d673..720c4f268 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,13 +4,16 @@ MAINTAINER Stefan Kögl # use bash instead of built-in sh, because it does not support "source" during build RUN rm /bin/sh && ln -s /bin/bash /bin/sh -RUN apt-get update - -# install Docker dependencies -RUN apt-get install -y git virtualenv - -# install all packaged runtime dependencies -RUN yes | make install-deps +# install all packaged dependencies +RUN apt-get update && apt-get install -y \ + git \ + python2.7 \ + python2.7-dev \ + python-virtualenv \ + libpq-dev \ + libjpeg-dev \ + zlib1g-dev \ + libwebp-dev # create log directories RUN mkdir -p /var/log/gunicorn From 89d689ff35e4160821021ac3fb7bc186d9c40b42 Mon Sep 17 00:00:00 2001 From: Jean-Fred Date: Tue, 9 Aug 2016 12:21:07 +0100 Subject: [PATCH 05/30] Docker-compose setup --- Dockerfile | 25 +++++++-------------- bin/docker-env.sh | 8 ------- contrib/Dockerfile.postgres | 3 +++ contrib/docker.env | 3 +++ contrib/init-db.sql | 2 -- docker-compose.yml | 45 +++++++++++++++++++++++++++++++++++++ fig.yml | 2 -- mygpo/celeryconfig.py | 4 ++++ mygpo/settings.py | 3 +-- 9 files changed, 64 insertions(+), 31 deletions(-) create mode 100644 contrib/Dockerfile.postgres create mode 100644 contrib/docker.env create mode 100644 docker-compose.yml delete mode 100644 fig.yml create mode 100644 mygpo/celeryconfig.py diff --git a/Dockerfile b/Dockerfile index 720c4f268..194b4ee25 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,9 @@ -FROM ubuntu:14.04 +FROM python:3 MAINTAINER Stefan Kögl -# use bash instead of built-in sh, because it does not support "source" during build -RUN rm /bin/sh && ln -s /bin/bash /bin/sh - # install all packaged dependencies RUN apt-get update && apt-get install -y \ git \ - python2.7 \ - python2.7-dev \ - python-virtualenv \ libpq-dev \ libjpeg-dev \ zlib1g-dev \ @@ -18,24 +12,21 @@ RUN apt-get update && apt-get install -y \ # create log directories RUN mkdir -p /var/log/gunicorn -# copy source -COPY . /srv/mygpo WORKDIR /srv/mygpo -# run everything in a virtualenv -RUN virtualenv venv -RUN source venv/bin/activate +COPY requirements.txt . +COPY requirements-setup.txt . # install all runtime dependencies RUN pip install \ - -r /srv/mygpo/requirements.txt \ - -r /srv/mygpo/requirements-setup.txt + -r requirements.txt \ + -r requirements-setup.txt + +# copy source +COPY . . # set up missing environment variables ENTRYPOINT ["/srv/mygpo/bin/docker-env.sh"] # default launch command CMD ["gunicorn", "mygpo.wsgi:application", "--access-logfile", "-"] - -# HTTP port -EXPOSE 8000 diff --git a/bin/docker-env.sh b/bin/docker-env.sh index 3d7ca6aef..5c5969cf8 100755 --- a/bin/docker-env.sh +++ b/bin/docker-env.sh @@ -1,12 +1,4 @@ #!/bin/bash -# Docker doesn't have a great way to set environment variables at startup. -# This scripts will set up some defaults. - -# if a DATABSE_URL is provided from outside, use it -if [[ -z "$DATABASE_URL" ]]; then - # otherwise construct one using a linked "db" container - export DATABASE_URL="postgres://mygpo:mygpo@${DB_PORT_5432_TCP_ADDR}:5432/mygpo" -fi # if not SECRET_KEY is provided from outside, create a random one if [[ -z "$SECRET_KEY" ]]; then diff --git a/contrib/Dockerfile.postgres b/contrib/Dockerfile.postgres new file mode 100644 index 000000000..9d674117e --- /dev/null +++ b/contrib/Dockerfile.postgres @@ -0,0 +1,3 @@ +FROM postgres:9.5 + +COPY init-db.sql /docker-entrypoint-initdb.d/ diff --git a/contrib/docker.env b/contrib/docker.env new file mode 100644 index 000000000..bfa816d3e --- /dev/null +++ b/contrib/docker.env @@ -0,0 +1,3 @@ +BROKER_URL=redis://root@redis:6379/0 +ELASTICSEARCH_SERVER=es:9200/ +DATABASE_URL=postgresql://mygpo:mygpo@postgres:5432/mygpo diff --git a/contrib/init-db.sql b/contrib/init-db.sql index 5b4dad338..c74177793 100644 --- a/contrib/init-db.sql +++ b/contrib/init-db.sql @@ -1,6 +1,4 @@ -CREATE USER mygpo WITH PASSWORD 'mygpo'; ALTER USER mygpo CREATEDB; -- required for creating test database -CREATE DATABASE mygpo; CREATE DATABASE test_mygpo; GRANT ALL PRIVILEGES ON DATABASE mygpo to mygpo; GRANT ALL PRIVILEGES ON DATABASE test_mygpo to mygpo; diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..54228d143 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,45 @@ + +version: '2' + +services: + + base: + build: + context: . + dockerfile: Dockerfile + env_file: contrib/docker.env + volumes: + - .:/srv/mygpo + - ./contrib/docker.env:/docker.env + + postgres: + build: + context: contrib + dockerfile: Dockerfile.postgres + environment: + POSTGRES_USER: mygpo + POSTGRES_PASSWORD: mygpo + + memcached: + image: memcached:1.4 + + redis: + image: redis:3.2-alpine + + web: + extends: base + command: python ./manage.py runserver 0.0.0.0:8000 + links: + - redis + - postgres + - memcached + ports: + - '8000:8000' + + worker: + extends: base + command: python manage.py celery beat -S djcelery.schedulers.DatabaseScheduler + links: + - web + - postgres + - redis diff --git a/fig.yml b/fig.yml deleted file mode 100644 index d13cdd25e..000000000 --- a/fig.yml +++ /dev/null @@ -1,2 +0,0 @@ -db: - image: postgres diff --git a/mygpo/celeryconfig.py b/mygpo/celeryconfig.py new file mode 100644 index 000000000..e69e78ba2 --- /dev/null +++ b/mygpo/celeryconfig.py @@ -0,0 +1,4 @@ +import os + +BROKER_URL = os.environ['BROKER_URL'] +CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', BROKER_URL) diff --git a/mygpo/settings.py b/mygpo/settings.py index a9e4e7b72..7ff6de987 100644 --- a/mygpo/settings.py +++ b/mygpo/settings.py @@ -34,8 +34,7 @@ def get_intOrNone(name, default): MANAGERS = ADMINS DATABASES = { - 'default': dj_database_url.config( - default='postgres://mygpo:mygpo@localhost/mygpo'), + 'default': dj_database_url.parse(os.environ['DATABASE_URL']) } From cdcb34f8b83144c57f973fd4ab798539ab85c207 Mon Sep 17 00:00:00 2001 From: Jean-Fred Date: Sat, 1 Jul 2017 16:46:39 +0100 Subject: [PATCH 06/30] Serve static assets using Whitenoise This is useful for development, when there is no nginx server to serve assets. --- mygpo/settings.py | 3 ++- requirements.txt | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/mygpo/settings.py b/mygpo/settings.py index 7ff6de987..5675bcc25 100644 --- a/mygpo/settings.py +++ b/mygpo/settings.py @@ -67,7 +67,7 @@ def get_intOrNone(name, default): # to load the internationalization machinery. USE_I18N = True -STATIC_ROOT = 'staticfiles' +STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles') STATIC_URL = '/media/' STATICFILES_DIRS = ( @@ -112,6 +112,7 @@ def get_intOrNone(name, default): MIDDLEWARE_CLASSES = ( + 'whitenoise.middleware.WhiteNoiseMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', diff --git a/requirements.txt b/requirements.txt index 5233f8727..15529061b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,4 @@ python-dateutil==2.6.0 redis==2.10.5 django-celery==3.2.1 requests==2.18.1 +whitenoise==3.3.0 From 44e40e4e0242b4a24dc96d31506aad5b215ba83f Mon Sep 17 00:00:00 2001 From: Jean-Fred Date: Sat, 1 Jul 2017 16:47:14 +0100 Subject: [PATCH 07/30] Avoid crash during initial migration As suggested by @elelay on https://github.com/gpodder/mygpo/pull/32#issuecomment-292723438 --- mygpo/users/checks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mygpo/users/checks.py b/mygpo/users/checks.py index 61d730723..8fd90fb7e 100644 --- a/mygpo/users/checks.py +++ b/mygpo/users/checks.py @@ -1,6 +1,6 @@ from django.core.checks import register, Warning from django.db import connection -from django.db.utils import OperationalError +from django.db.utils import OperationalError, ProgrammingError SQL = """ @@ -29,8 +29,8 @@ def check_case_insensitive_users(app_configs=None, **kwargs): wid = 'users.W001' errors.append(Warning(txt, id=wid)) - except OperationalError as oe: - if 'no such table: auth_user' in str(oe): + except (OperationalError, ProgrammingError) as oe: + if 'no such table: auth_user' in str(oe) or 'relation "auth_user" does not exist' in str(oe): # Ignore if the table does not yet exist, eg when initally # running ``manage.py migrate`` pass From 6f74bbd08e1e62aa32191a668d76368a5e88d835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20K=C3=B6gl?= Date: Sun, 23 Jul 2017 11:51:59 +0200 Subject: [PATCH 08/30] Update Dockerfile, makefile to fix build --- Dockerfile | 16 ++++++++-------- makefile | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index d5fd8d673..c18da48ef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:14.04 +FROM ubuntu:latest MAINTAINER Stefan Kögl # use bash instead of built-in sh, because it does not support "source" during build @@ -7,7 +7,11 @@ RUN rm /bin/sh && ln -s /bin/bash /bin/sh RUN apt-get update # install Docker dependencies -RUN apt-get install -y git virtualenv +RUN apt-get install -y git virtualenv make + +# copy source +COPY . /srv/mygpo +WORKDIR /srv/mygpo # install all packaged runtime dependencies RUN yes | make install-deps @@ -15,16 +19,12 @@ RUN yes | make install-deps # create log directories RUN mkdir -p /var/log/gunicorn -# copy source -COPY . /srv/mygpo -WORKDIR /srv/mygpo - # run everything in a virtualenv -RUN virtualenv venv +RUN virtualenv -p `which python3` venv RUN source venv/bin/activate # install all runtime dependencies -RUN pip install \ +RUN venv/bin/pip install \ -r /srv/mygpo/requirements.txt \ -r /srv/mygpo/requirements-setup.txt diff --git a/makefile b/makefile index 3b7dcb138..0c7bfc61c 100644 --- a/makefile +++ b/makefile @@ -18,8 +18,8 @@ clean: git clean -fX install-deps: - sudo apt-get install libpq-dev libjpeg-dev zlib1g-dev libwebp-dev \ - build-essential python3-dev + apt-get install libpq-dev libjpeg-dev zlib1g-dev libwebp-dev \ + build-essential python3-dev make docker-build: sudo docker build -t="mygpo/web" . From d3580b3a88a61d6ab408db8448d2c9a295bd625d Mon Sep 17 00:00:00 2001 From: Jean-Fred Date: Sun, 23 Jul 2017 11:03:42 +0100 Subject: [PATCH 09/30] Comment out django-nose from INSTALLED_APPS This keeps crashing with ` ModuleNotFoundError: No module named 'django_nose'` even though the package is installed. Let's ignore for now. --- mygpo/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mygpo/settings.py b/mygpo/settings.py index 5675bcc25..0c110b463 100644 --- a/mygpo/settings.py +++ b/mygpo/settings.py @@ -157,7 +157,7 @@ def get_intOrNone(name, default): 'mygpo.pubsub', 'mygpo.podcastlists', 'mygpo.votes', - 'django_nose', + # 'django_nose', ] try: From 20ff6fa7a75c4b2bfd76343cc4d1b9a70b766d0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20K=C3=B6gl?= Date: Mon, 24 Jul 2017 21:51:34 +0200 Subject: [PATCH 10/30] Import django-nose only when it is available --- mygpo/settings.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/mygpo/settings.py b/mygpo/settings.py index a9e4e7b72..910160478 100644 --- a/mygpo/settings.py +++ b/mygpo/settings.py @@ -157,9 +157,9 @@ def get_intOrNone(name, default): 'mygpo.pubsub', 'mygpo.podcastlists', 'mygpo.votes', - 'django_nose', ] + try: import debug_toolbar INSTALLED_APPS += ['debug_toolbar'] @@ -394,10 +394,20 @@ def get_intOrNone(name, default): PODCAST_AD_ID = os.getenv('PODCAST_AD_ID') -TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' +try: + import django_nose + + INSTALLED_APPS += [ + 'django_nose', + ] -NOSE_ARGS = [ - '--with-doctest', - '--stop', - '--where=mygpo', -] + TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' + + NOSE_ARGS = [ + '--with-doctest', + '--stop', + '--where=mygpo', + ] + +except ImportError: + print('django-nose is not installed.') From b7aadf5418cbef08ac7b1d781216cd2bca2ce75e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20K=C3=B6gl?= Date: Mon, 24 Jul 2017 21:51:59 +0200 Subject: [PATCH 11/30] Run collectstatic, compilemessages in Docker build --- Dockerfile | 5 +++++ makefile | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c18da48ef..da00d14a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,6 +28,11 @@ RUN venv/bin/pip install \ -r /srv/mygpo/requirements.txt \ -r /srv/mygpo/requirements-setup.txt +ENV SECRET_KEY temp + +RUN venv/bin/python manage.py collectstatic --no-input +RUN venv/bin/python manage.py compilemessages + # set up missing environment variables ENTRYPOINT ["/srv/mygpo/bin/docker-env.sh"] diff --git a/makefile b/makefile index 0c7bfc61c..919623976 100644 --- a/makefile +++ b/makefile @@ -19,7 +19,7 @@ clean: install-deps: apt-get install libpq-dev libjpeg-dev zlib1g-dev libwebp-dev \ - build-essential python3-dev make + build-essential python3-dev make gettext docker-build: sudo docker build -t="mygpo/web" . From 95bf71464c3dfab5fe6c1f46960350a65735fe3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20K=C3=B6gl?= Date: Tue, 25 Jul 2017 18:28:44 +0200 Subject: [PATCH 12/30] Revert some changes from 89d689ff35e4160821021ac3f The now-reverted changed made the use of the scripts less generic for other uses. --- contrib/init-db.sql | 2 ++ docker-compose.yml | 3 --- mygpo/celeryconfig.py | 4 ---- 3 files changed, 2 insertions(+), 7 deletions(-) delete mode 100644 mygpo/celeryconfig.py diff --git a/contrib/init-db.sql b/contrib/init-db.sql index c74177793..5b4dad338 100644 --- a/contrib/init-db.sql +++ b/contrib/init-db.sql @@ -1,4 +1,6 @@ +CREATE USER mygpo WITH PASSWORD 'mygpo'; ALTER USER mygpo CREATEDB; -- required for creating test database +CREATE DATABASE mygpo; CREATE DATABASE test_mygpo; GRANT ALL PRIVILEGES ON DATABASE mygpo to mygpo; GRANT ALL PRIVILEGES ON DATABASE test_mygpo to mygpo; diff --git a/docker-compose.yml b/docker-compose.yml index 54228d143..2976dd99b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,9 +16,6 @@ services: build: context: contrib dockerfile: Dockerfile.postgres - environment: - POSTGRES_USER: mygpo - POSTGRES_PASSWORD: mygpo memcached: image: memcached:1.4 diff --git a/mygpo/celeryconfig.py b/mygpo/celeryconfig.py deleted file mode 100644 index e69e78ba2..000000000 --- a/mygpo/celeryconfig.py +++ /dev/null @@ -1,4 +0,0 @@ -import os - -BROKER_URL = os.environ['BROKER_URL'] -CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', BROKER_URL) From ea99b1f4d80b0d953f4de7523cd2c36a2083ae3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20K=C3=B6gl?= Date: Tue, 25 Jul 2017 20:11:37 +0200 Subject: [PATCH 13/30] Add container for Celery worker --- docker-compose.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 2976dd99b..4f16eba18 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,6 +34,17 @@ services: - '8000:8000' worker: + extends: base + command: python manage.py celery worker + links: + - web + - postgres + - redis + depends_on: + - postgres + - redis + + beat: extends: base command: python manage.py celery beat -S djcelery.schedulers.DatabaseScheduler links: From 558179bf7290201f4eb11da8a8f905253f4cba38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20K=C3=B6gl?= Date: Tue, 25 Jul 2017 20:12:04 +0200 Subject: [PATCH 14/30] Add dependencies between containers --- docker-compose.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 4f16eba18..2d313591e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,6 +32,9 @@ services: - memcached ports: - '8000:8000' + depends_on: + - postgres + - redis worker: extends: base @@ -51,3 +54,6 @@ services: - web - postgres - redis + depends_on: + - postgres + - redis From bcf227e9089e998c57ffde9bcb435c8e74f86971 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20K=C3=B6gl?= Date: Tue, 25 Jul 2017 20:12:24 +0200 Subject: [PATCH 15/30] Run gunicorn webserver `manage.py runserver` is not intended for serving production traffic https://docs.djangoproject.com/en/1.8/ref/django-admin/#runserver-port-or-address-port --- Dockerfile | 4 +++- docker-compose.yml | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index adc6fb318..45763c540 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,4 +34,6 @@ RUN python manage.py compilemessages ENTRYPOINT ["/srv/mygpo/bin/docker-env.sh"] # default launch command -CMD ["gunicorn", "mygpo.wsgi:application", "--access-logfile", "-"] +CMD ["gunicorn", "mygpo.wsgi:application", "--access-logfile", "-", "--bind=0.0.0.0:8000"] + +EXPOSE 8000 diff --git a/docker-compose.yml b/docker-compose.yml index 2d313591e..2969e8d38 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,7 +25,6 @@ services: web: extends: base - command: python ./manage.py runserver 0.0.0.0:8000 links: - redis - postgres From 14915829013a81f08352c71502beb7e2c1d467ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20K=C3=B6gl?= Date: Mon, 25 Sep 2017 21:41:50 +0200 Subject: [PATCH 16/30] Add missing setting SEARCH_CUTOFF --- mygpo/settings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mygpo/settings.py b/mygpo/settings.py index 28375d75e..b0491c061 100644 --- a/mygpo/settings.py +++ b/mygpo/settings.py @@ -406,3 +406,6 @@ def get_intOrNone(name, default): except ImportError: print('django-nose is not installed.') + + +SEARCH_CUTOFF = float(os.getenv('SEARCH_CUTOFF', 0.3)) From 0763228d582bfec66ff4307877a2bca84ec07a28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20K=C3=B6gl?= Date: Sat, 28 Oct 2017 21:15:28 +0200 Subject: [PATCH 17/30] Remove references to (unused) Elasticsearch --- contrib/docker.env | 1 - doc/ops/docker.rst | 4 ---- 2 files changed, 5 deletions(-) diff --git a/contrib/docker.env b/contrib/docker.env index bfa816d3e..a70e9c0af 100644 --- a/contrib/docker.env +++ b/contrib/docker.env @@ -1,3 +1,2 @@ BROKER_URL=redis://root@redis:6379/0 -ELASTICSEARCH_SERVER=es:9200/ DATABASE_URL=postgresql://mygpo:mygpo@postgres:5432/mygpo diff --git a/doc/ops/docker.rst b/doc/ops/docker.rst index 9ee0907b1..6e5fa65f5 100644 --- a/doc/ops/docker.rst +++ b/doc/ops/docker.rst @@ -37,10 +37,6 @@ Initialize the tables. This needs needs to be run for every update. :: sudo docker run --rm --link db:db -e SECRET_KEY=asdf mygpo/web python manage.py migrate -Elasticsearch -------------- - - Redis ----- From a6c9430c61650e5720daa869ec1082877c2a0978 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20K=C3=B6gl?= Date: Sat, 28 Oct 2017 21:15:46 +0200 Subject: [PATCH 18/30] Update PostgreSQL container to version 10 --- contrib/Dockerfile.postgres | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/Dockerfile.postgres b/contrib/Dockerfile.postgres index 9d674117e..4ac2371d2 100644 --- a/contrib/Dockerfile.postgres +++ b/contrib/Dockerfile.postgres @@ -1,3 +1,3 @@ -FROM postgres:9.5 +FROM postgres:10 COPY init-db.sql /docker-entrypoint-initdb.d/ From 7a46e2d2373876ee0a4225663e862c64e5b43904 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20K=C3=B6gl?= Date: Tue, 16 Jan 2018 16:29:32 +0100 Subject: [PATCH 19/30] Update commands in celery containers --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2969e8d38..44d17666f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,7 @@ services: worker: extends: base - command: python manage.py celery worker + command: celery -A mygpo worker --concurrency=3 -l info -Ofair links: - web - postgres @@ -48,7 +48,7 @@ services: beat: extends: base - command: python manage.py celery beat -S djcelery.schedulers.DatabaseScheduler + command: celery -A mygpo beat --pidfile /tmp/celerybeat.pid -S django links: - web - postgres From 57a4d0772153b65e8c61676edd19cfcf5ef62f7a Mon Sep 17 00:00:00 2001 From: Eric Le Lay Date: Sun, 14 Jan 2018 19:30:01 +0100 Subject: [PATCH 20/30] wait-for-postgres.sh wrapper used in containers depending on postresql readiness. No need for full psql command: the simple psql.py script is enough. --- Dockerfile | 2 +- contrib/psql.py | 14 ++++++++++++++ contrib/wait-for-postgres.sh | 14 ++++++++++++++ docker-compose.yml | 4 ++-- 4 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 contrib/psql.py create mode 100755 contrib/wait-for-postgres.sh diff --git a/Dockerfile b/Dockerfile index 45763c540..3d08fde55 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,6 +34,6 @@ RUN python manage.py compilemessages ENTRYPOINT ["/srv/mygpo/bin/docker-env.sh"] # default launch command -CMD ["gunicorn", "mygpo.wsgi:application", "--access-logfile", "-", "--bind=0.0.0.0:8000"] +CMD ["/srv/mygpo/contrib/wait-for-postgres.sh", "gunicorn", "mygpo.wsgi:application", "--access-logfile", "-", "--bind=0.0.0.0:8000"] EXPOSE 8000 diff --git a/contrib/psql.py b/contrib/psql.py new file mode 100644 index 000000000..11e0256aa --- /dev/null +++ b/contrib/psql.py @@ -0,0 +1,14 @@ +import os +import sys + +import psycopg2cffi as psycopg2 + + +if __name__ == "__main__": + dburl = os.environ["DATABASE_URL"] + print("Trying to connect to {}".format(dburl)) + try: + conn = psycopg2.connect(dburl) + except: + print("pg connect failed, try later") + sys.exit(-1) \ No newline at end of file diff --git a/contrib/wait-for-postgres.sh b/contrib/wait-for-postgres.sh new file mode 100755 index 000000000..c182ef4d3 --- /dev/null +++ b/contrib/wait-for-postgres.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# wait-for-postgres.sh + +set -e + +cmd="$@" + +until python contrib/psql.py; do + >&2 echo "Postgres is unavailable - sleeping" + sleep 1 +done + +>&2 echo "Postgres is up - executing command $cmd" +exec $cmd diff --git a/docker-compose.yml b/docker-compose.yml index 44d17666f..f31a8566f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,7 @@ services: worker: extends: base - command: celery -A mygpo worker --concurrency=3 -l info -Ofair + command: /srv/mygpo/contrib/wait-for-postgres.sh celery -A mygpo worker --concurrency=3 -l info -Ofair links: - web - postgres @@ -48,7 +48,7 @@ services: beat: extends: base - command: celery -A mygpo beat --pidfile /tmp/celerybeat.pid -S django + command: /srv/mygpo/contrib/wait-for-postgres.sh celery -A mygpo beat --pidfile /tmp/celerybeat.pid -S django links: - web - postgres From aef345a2e5ef1651200087b34d334e08828f805c Mon Sep 17 00:00:00 2001 From: Eric Le Lay Date: Fri, 26 Jan 2018 12:18:00 +0100 Subject: [PATCH 21/30] docker-compose: wait for postgres + init; don't run base - enhanced wait-for-postgres.py: 1. waits for DB 2. if not running migrate, exit with error message if tables are missing - move base to a different file otherwise it's also started and two web services are running. --- Dockerfile | 2 +- contrib/psql.py | 14 ---------- contrib/wait-for-postgres.py | 51 ++++++++++++++++++++++++++++++++++++ contrib/wait-for-postgres.sh | 14 ---------- docker-compose-common.yml | 14 ++++++++++ docker-compose.yml | 25 ++++++++---------- 6 files changed, 77 insertions(+), 43 deletions(-) delete mode 100644 contrib/psql.py create mode 100755 contrib/wait-for-postgres.py delete mode 100755 contrib/wait-for-postgres.sh create mode 100644 docker-compose-common.yml diff --git a/Dockerfile b/Dockerfile index 3d08fde55..1aa9d676b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,6 +34,6 @@ RUN python manage.py compilemessages ENTRYPOINT ["/srv/mygpo/bin/docker-env.sh"] # default launch command -CMD ["/srv/mygpo/contrib/wait-for-postgres.sh", "gunicorn", "mygpo.wsgi:application", "--access-logfile", "-", "--bind=0.0.0.0:8000"] +CMD ["/srv/mygpo/contrib/wait-for-postgres.py", "gunicorn", "mygpo.wsgi:application", "--access-logfile", "-", "--bind=0.0.0.0:8000"] EXPOSE 8000 diff --git a/contrib/psql.py b/contrib/psql.py deleted file mode 100644 index 11e0256aa..000000000 --- a/contrib/psql.py +++ /dev/null @@ -1,14 +0,0 @@ -import os -import sys - -import psycopg2cffi as psycopg2 - - -if __name__ == "__main__": - dburl = os.environ["DATABASE_URL"] - print("Trying to connect to {}".format(dburl)) - try: - conn = psycopg2.connect(dburl) - except: - print("pg connect failed, try later") - sys.exit(-1) \ No newline at end of file diff --git a/contrib/wait-for-postgres.py b/contrib/wait-for-postgres.py new file mode 100755 index 000000000..9adc421e6 --- /dev/null +++ b/contrib/wait-for-postgres.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +import os +import sys +import time + +import psycopg2cffi as psycopg2 + + +if __name__ == "__main__": + dburl = os.environ["DATABASE_URL"] + print("Trying to connect to {}".format(dburl)) + conn = None + while not conn: + try: + conn = psycopg2.connect(dburl) + except: + print("Postgres is unavailable - sleeping") + time.sleep(1) + + cur = conn.cursor() + cur.execute("""SELECT EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = 'votes_vote' + )""") + has_table = cur.fetchone()[0] + cur.close() + conn.close() + + if 'migrate' in sys.argv: + if has_table: + print("Database already initialized, exiting") + sys.exit(0) + else: + print("Postgres is available => will initialize") + else: + if not has_table: + print("ERROR: Postgres is available but not initialized.\n" + "Please run:\n" + "\tdocker-compose run web /srv/mygpo/contrib/wait-for-postgres.py python manage.py migrate\n" + "and restart.") + sys.exit(-1) + + if sys.argv[1]: + cmd = sys.argv[1:] + print("Postgres is up - executing command {}".format(" ".join(cmd))) + sys.stdout.flush() + os.execvp(cmd[0], cmd) + else: + print("Postgres is up - no command given, exiting") \ No newline at end of file diff --git a/contrib/wait-for-postgres.sh b/contrib/wait-for-postgres.sh deleted file mode 100755 index c182ef4d3..000000000 --- a/contrib/wait-for-postgres.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# wait-for-postgres.sh - -set -e - -cmd="$@" - -until python contrib/psql.py; do - >&2 echo "Postgres is unavailable - sleeping" - sleep 1 -done - ->&2 echo "Postgres is up - executing command $cmd" -exec $cmd diff --git a/docker-compose-common.yml b/docker-compose-common.yml new file mode 100644 index 000000000..e84cbf4e4 --- /dev/null +++ b/docker-compose-common.yml @@ -0,0 +1,14 @@ + +version: '2' + +services: + + base: + build: + context: . + dockerfile: Dockerfile + env_file: contrib/docker.env + volumes: + - .:/srv/mygpo + - ./contrib/docker.env:/docker.env + diff --git a/docker-compose.yml b/docker-compose.yml index f31a8566f..5311f565e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,15 +3,6 @@ version: '2' services: - base: - build: - context: . - dockerfile: Dockerfile - env_file: contrib/docker.env - volumes: - - .:/srv/mygpo - - ./contrib/docker.env:/docker.env - postgres: build: context: contrib @@ -24,7 +15,9 @@ services: image: redis:3.2-alpine web: - extends: base + extends: + file: docker-compose-common.yml + service: base links: - redis - postgres @@ -36,8 +29,10 @@ services: - redis worker: - extends: base - command: /srv/mygpo/contrib/wait-for-postgres.sh celery -A mygpo worker --concurrency=3 -l info -Ofair + extends: + file: docker-compose-common.yml + service: base + command: /srv/mygpo/contrib/wait-for-postgres.py celery -A mygpo worker --concurrency=3 -l info -Ofair links: - web - postgres @@ -47,8 +42,10 @@ services: - redis beat: - extends: base - command: /srv/mygpo/contrib/wait-for-postgres.sh celery -A mygpo beat --pidfile /tmp/celerybeat.pid -S django + extends: + file: docker-compose-common.yml + service: base + command: /srv/mygpo/contrib/wait-for-postgres.py celery -A mygpo beat --pidfile /tmp/celerybeat.pid -S django links: - web - postgres From 7611dbca66068284573d49e67271413371264317 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20K=C3=B6gl?= Date: Wed, 15 Aug 2018 18:27:12 +0200 Subject: [PATCH 22/30] Specify Python 3.6 in Dockerfile Python 3.7 is not yet supported --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1aa9d676b..3244e345a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3 +FROM python:3.6 MAINTAINER Stefan Kögl RUN apt-get update && apt-get install -y git make From a15262e24fc683382cdd1c54a464c17375448a65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20K=C3=B6gl?= Date: Fri, 18 Oct 2019 19:53:39 +0000 Subject: [PATCH 23/30] Fix code formatting --- mygpo/settings.py | 4 +--- mygpo/users/checks.py | 4 +++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mygpo/settings.py b/mygpo/settings.py index a21ddb8af..0f7f0b51f 100644 --- a/mygpo/settings.py +++ b/mygpo/settings.py @@ -235,8 +235,7 @@ def get_intOrNone(name, default): GOOGLE_ANALYTICS_PROPERTY_ID = os.getenv('GOOGLE_ANALYTICS_PROPERTY_ID', '') -DIRECTORY_EXCLUDED_TAGS = parse_strlist(os.getenv('DIRECTORY_EXCLUDED_TAGS', - '')) +DIRECTORY_EXCLUDED_TAGS = parse_strlist(os.getenv('DIRECTORY_EXCLUDED_TAGS', '')) FLICKR_API_KEY = os.getenv('FLICKR_API_KEY', '') @@ -247,7 +246,6 @@ def get_intOrNone(name, default): MAINTENANCE = get_bool('MAINTENANCE', False) - ALLOWED_HOSTS = ['*'] diff --git a/mygpo/users/checks.py b/mygpo/users/checks.py index 26624eb38..34ca07593 100644 --- a/mygpo/users/checks.py +++ b/mygpo/users/checks.py @@ -30,7 +30,9 @@ def check_case_insensitive_users(app_configs=None, **kwargs): errors.append(Warning(txt, id=wid)) except (OperationalError, ProgrammingError) as oe: - if 'no such table: auth_user' in str(oe) or 'relation "auth_user" does not exist' in str(oe): + if 'no such table: auth_user' in str( + oe + ) or 'relation "auth_user" does not exist' in str(oe): # Ignore if the table does not yet exist, eg when initally # running ``manage.py migrate`` pass From 31365aa93e73a11434f2f4b294faef36fde74875 Mon Sep 17 00:00:00 2001 From: Ssuching Yu Date: Mon, 24 Feb 2020 16:09:27 +0800 Subject: [PATCH 24/30] Fix makefile tab indentation. --- makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefile b/makefile index 3fb52097a..76cbda4e3 100644 --- a/makefile +++ b/makefile @@ -34,7 +34,7 @@ clean: git clean -fX install-deps: - sudo apt-get install libpq-dev libjpeg-dev zlib1g-dev libwebp-dev \ + sudo apt-get install libpq-dev libjpeg-dev zlib1g-dev libwebp-dev \ build-essential python3-dev virtualenv libffi-dev redis postgresql docker-build: From 406f1d54f2fe4fb4f7c4161b82574baa3f9d2e5e Mon Sep 17 00:00:00 2001 From: Ssuching Yu Date: Mon, 24 Feb 2020 16:28:33 +0800 Subject: [PATCH 25/30] Upgrade whitenoise to fix django.utils.six ModuleNotFoundError --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 98866c450..603ef5679 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,5 +16,5 @@ redis==3.3.11 django-celery-results==1.2.0 django-celery-beat==1.5.0 requests==2.22.0 -whitenoise==3.3.0 +whitenoise==5.0.1 django-db-geventpool==3.1.0 From 8484c3ee3dbb8c404d1dc282cc3aecbb58a189c2 Mon Sep 17 00:00:00 2001 From: Jean-Frederic Date: Tue, 14 Apr 2020 12:50:57 +0200 Subject: [PATCH 26/30] Allow to run Makefile command without sudo There is no sudo available in the Docker environment. Let's just move the `apt-get` call command to a variable, and override it in Makefile call in the Dockerfile. --- Dockerfile | 2 +- makefile | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3244e345a..2e6180b2f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ WORKDIR /srv/mygpo COPY makefile . # install all packaged runtime dependencies -RUN yes | make install-deps +RUN yes | make install-deps APT=apt-get # create log directories RUN mkdir -p /var/log/gunicorn diff --git a/makefile b/makefile index 76cbda4e3..3b80c2a42 100644 --- a/makefile +++ b/makefile @@ -1,3 +1,5 @@ +APT=sudo apt-get + all: help help: @@ -34,7 +36,7 @@ clean: git clean -fX install-deps: - sudo apt-get install libpq-dev libjpeg-dev zlib1g-dev libwebp-dev \ + $(APT) install libpq-dev libjpeg-dev zlib1g-dev libwebp-dev \ build-essential python3-dev virtualenv libffi-dev redis postgresql docker-build: From 4efb585aaa5b9f23b86f4844c1a59cbf6d7b21c9 Mon Sep 17 00:00:00 2001 From: Jean-Frederic Date: Tue, 14 Apr 2020 12:52:36 +0200 Subject: [PATCH 27/30] Set POSTGRES_HOST_AUTH_METHOD in Postgres Docker There was a breaking change recently in Postgres Docker image: ``` Error: Database is uninitialized and superuser password is not specified. You must specify POSTGRES_PASSWORD for the superuser. Use "-e POSTGRES_PASSWORD=password" to set it in "docker run". You may also use POSTGRES_HOST_AUTH_METHOD=trust to allow all connections without a password. This is *not* recommended. See PostgreSQL documentation about "trust": https://www.postgresql.org/docs/current/auth-trust.html ``` This may not be the best setting, but let's go for this. See https://github.com/docker-library/postgres/issues/681 --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 5311f565e..31ced40cd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,8 @@ services: build: context: contrib dockerfile: Dockerfile.postgres + environment: + POSTGRES_HOST_AUTH_METHOD: 'trust' memcached: image: memcached:1.4 From 93131ae891e47d190be56ac02301be95b335cddb Mon Sep 17 00:00:00 2001 From: Jean-Frederic Date: Tue, 14 Apr 2020 13:01:43 +0200 Subject: [PATCH 28/30] Add gettext to makefile dependencies Without it, the step `compilemessages` fails. --- makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/makefile b/makefile index 3b80c2a42..8fb5d893f 100644 --- a/makefile +++ b/makefile @@ -36,7 +36,7 @@ clean: git clean -fX install-deps: - $(APT) install libpq-dev libjpeg-dev zlib1g-dev libwebp-dev \ + $(APT) install libpq-dev libjpeg-dev zlib1g-dev libwebp-dev gettext \ build-essential python3-dev virtualenv libffi-dev redis postgresql docker-build: From 9f4ac3810d8235190a29dee157e5095ac15db32e Mon Sep 17 00:00:00 2001 From: Jean-Frederic Date: Tue, 14 Apr 2020 14:49:53 +0200 Subject: [PATCH 29/30] Fix variable in French locale --- mygpo/locale/fr/LC_MESSAGES/django.po | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mygpo/locale/fr/LC_MESSAGES/django.po b/mygpo/locale/fr/LC_MESSAGES/django.po index e968b0862..257bc6c64 100644 --- a/mygpo/locale/fr/LC_MESSAGES/django.po +++ b/mygpo/locale/fr/LC_MESSAGES/django.po @@ -813,7 +813,7 @@ msgstr "Podcast inconnu" #: mygpo/podcasts/models.py:700 #, python-brace-format msgid "Unknown Podcast from {domain}" -msgstr "Podcast inconnu de {domaine}" +msgstr "Podcast inconnu de {domain}" #: mygpo/podcasts/templates/episode.html:85 #: mygpo/podcasts/templates/episodes.html:27 From a7045b94e6ce958992f73081fda0a09cbdf0da25 Mon Sep 17 00:00:00 2001 From: Jean-Frederic Date: Tue, 14 Apr 2020 15:16:50 +0200 Subject: [PATCH 30/30] Set STATIC_ROOT for serving static assets using Whitenoise This is one of the few requirements for whitenoise to just work. See http://whitenoise.evans.io/en/stable/django.html (This had been done in cdcb34f8b, but probably lost in a master merge) --- mygpo/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mygpo/settings.py b/mygpo/settings.py index 393d22b20..648f8b5dc 100644 --- a/mygpo/settings.py +++ b/mygpo/settings.py @@ -86,7 +86,7 @@ def get_intOrNone(name, default): # Static Files -STATIC_ROOT = 'staticfiles' +STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles') STATIC_URL = '/static/' STATICFILES_DIRS = (os.path.abspath(os.path.join(BASE_DIR, '..', 'static')),)