diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..fa795eb6c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +.git +htdocs/media/logo +mygpo/settings_prod.py +doc/ +res/ +tools/ +locale/ +venv* +*.pyc +*/*.pyc +*/*/*.pyc +*/*/*/*.pyc diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..2e6180b2f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,39 @@ +FROM python:3.6 +MAINTAINER Stefan Kögl + +RUN apt-get update && apt-get install -y git make + +# copy source +WORKDIR /srv/mygpo + +COPY makefile . + +# install all packaged runtime dependencies +RUN yes | make install-deps APT=apt-get + +# create log directories +RUN mkdir -p /var/log/gunicorn + +COPY requirements.txt . +COPY requirements-setup.txt . + +# install all runtime dependencies +RUN pip install \ + -r requirements.txt \ + -r requirements-setup.txt + +# copy source +COPY . . + +# Post-deployment actions +ENV SECRET_KEY temp +RUN python manage.py collectstatic --no-input +RUN python manage.py compilemessages + +# set up missing environment variables +ENTRYPOINT ["/srv/mygpo/bin/docker-env.sh"] + +# default launch command +CMD ["/srv/mygpo/contrib/wait-for-postgres.py", "gunicorn", "mygpo.wsgi:application", "--access-logfile", "-", "--bind=0.0.0.0:8000"] + +EXPOSE 8000 diff --git a/bin/docker-env.sh b/bin/docker-env.sh new file mode 100755 index 000000000..347fa89a8 --- /dev/null +++ b/bin/docker-env.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# Docker doesn't have a great way to set environment variables at startup. +# This scripts will set up some defaults. + +# if a DATABSE_URL is provided from outside, use it +if [[ -z "$DATABASE_URL" ]]; then + # otherwise construct one using a linked "db" container + export DATABASE_URL="postgres://mygpo:mygpo@${DB_PORT_5432_TCP_ADDR}:5432/mygpo" +fi + +# if not SECRET_KEY is provided from outside, create a random one +if [[ -z "$SECRET_KEY" ]]; then + export SECRET_KEY=$(cat /dev/urandom | tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1) +fi + +# Execute the commands passed to this script +exec "$@" diff --git a/contrib/Dockerfile.postgres b/contrib/Dockerfile.postgres new file mode 100644 index 000000000..4ac2371d2 --- /dev/null +++ b/contrib/Dockerfile.postgres @@ -0,0 +1,3 @@ +FROM postgres:10 + +COPY init-db.sql /docker-entrypoint-initdb.d/ diff --git a/contrib/docker.env b/contrib/docker.env new file mode 100644 index 000000000..a70e9c0af --- /dev/null +++ b/contrib/docker.env @@ -0,0 +1,2 @@ +BROKER_URL=redis://root@redis:6379/0 +DATABASE_URL=postgresql://mygpo:mygpo@postgres:5432/mygpo diff --git a/contrib/init-db.sql b/contrib/init-db.sql new file mode 100644 index 000000000..5b4dad338 --- /dev/null +++ b/contrib/init-db.sql @@ -0,0 +1,9 @@ +CREATE USER mygpo WITH PASSWORD 'mygpo'; +ALTER USER mygpo CREATEDB; -- required for creating test database +CREATE DATABASE mygpo; +CREATE DATABASE test_mygpo; +GRANT ALL PRIVILEGES ON DATABASE mygpo to mygpo; +GRANT ALL PRIVILEGES ON DATABASE test_mygpo to mygpo; +ALTER DATABASE mygpo OWNER TO mygpo; +ALTER DATABASE test_mygpo OWNER TO mygpo; +ALTER ROLE mygpo SET statement_timeout = 5000; diff --git a/contrib/wait-for-postgres.py b/contrib/wait-for-postgres.py new file mode 100755 index 000000000..9adc421e6 --- /dev/null +++ b/contrib/wait-for-postgres.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +import os +import sys +import time + +import psycopg2cffi as psycopg2 + + +if __name__ == "__main__": + dburl = os.environ["DATABASE_URL"] + print("Trying to connect to {}".format(dburl)) + conn = None + while not conn: + try: + conn = psycopg2.connect(dburl) + except: + print("Postgres is unavailable - sleeping") + time.sleep(1) + + cur = conn.cursor() + cur.execute("""SELECT EXISTS ( + SELECT 1 + FROM information_schema.tables + WHERE table_schema = 'public' + AND table_name = 'votes_vote' + )""") + has_table = cur.fetchone()[0] + cur.close() + conn.close() + + if 'migrate' in sys.argv: + if has_table: + print("Database already initialized, exiting") + sys.exit(0) + else: + print("Postgres is available => will initialize") + else: + if not has_table: + print("ERROR: Postgres is available but not initialized.\n" + "Please run:\n" + "\tdocker-compose run web /srv/mygpo/contrib/wait-for-postgres.py python manage.py migrate\n" + "and restart.") + sys.exit(-1) + + if sys.argv[1]: + cmd = sys.argv[1:] + print("Postgres is up - executing command {}".format(" ".join(cmd))) + sys.stdout.flush() + os.execvp(cmd[0], cmd) + else: + print("Postgres is up - no command given, exiting") \ No newline at end of file diff --git a/doc/dev/postgres-setup.rst b/doc/dev/postgres-setup.rst index 07f5b724b..738da0ad2 100644 --- a/doc/dev/postgres-setup.rst +++ b/doc/dev/postgres-setup.rst @@ -5,14 +5,6 @@ PostgreSQL Setup Use the following to set up a local PostgreSQL. -.. code-block:: sql - - CREATE USER mygpo WITH PASSWORD 'mygpo'; - ALTER USER mygpo CREATEDB; -- required for creating test database - CREATE DATABASE mygpo; - CREATE DATABASE test_mygpo; - GRANT ALL PRIVILEGES ON DATABASE mygpo to mygpo; - GRANT ALL PRIVILEGES ON DATABASE test_mygpo to mygpo; - ALTER DATABASE mygpo OWNER TO mygpo; - ALTER DATABASE test_mygpo OWNER TO mygpo; - ALTER ROLE mygpo SET statement_timeout = 5000; +.. literalinclude:: ../../contrib/init-db.sql + :language: sql + :linenos: diff --git a/doc/index.rst b/doc/index.rst index 3b750f539..b790bd34c 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -24,6 +24,7 @@ Contents publisher/index api/index dev/index + ops/index Translator Documentation diff --git a/doc/ops/configuration.rst b/doc/ops/configuration.rst new file mode 100644 index 000000000..9e4e32848 --- /dev/null +++ b/doc/ops/configuration.rst @@ -0,0 +1,106 @@ +Configuration +============= + +Configuration can be done through the following environment variables. + +``DEBUG`` +--------- +Debug mode shows error pages, enables debug output, etc. + + +``DATABASE_URL`` +---------------- +DB connection string in the form of ``postgres://USER:PASSWORD@HOST:PORT/NAME`` + + +``ACCOUNT_ACTIVATION_DAYS`` +--------------------------- +Number of days that newly registered users have time to activate their account. + + +``DEFAULT_FROM_EMAIL`` +---------------------- +Default sender address for outgoing emails. See `Django documentation +`__. + + +``SECRET_KEY`` +-------------- +See `Django documentation +`__. + + +``GOOGLE_ANALYTICS_PROPERTY_ID`` +-------------------------------- +Setting a Google Analytics Property ID activates GA integration. + + +``DIRECTORY_EXCLUDED_TAGS`` +--------------------------- +A comma-separated list of tags that are excluded from the directory. + + +``FLICKR_API_KEY`` +------------------ +Setting a Flickr API key activates Flickr integration. + + +``MAINTENANCE`` +--------------- +Switches the site into a maintenance mode. + + +* ``PODCAST_SLUG_SUBSCRIBER_LIMIT`` + +* ``MIN_SUBSCRIBERS_CATEGORY``: minimum number of subscribers that a podcast + needs to "push" one of its categories to the top + +* ``API_ACTIONS_MAX_NONBG``: maximum number of episode actions that the API + processes immediatelly before returning the response. Larger requests will + be handled in background. + +* ``ADSENSE_CLIENT`` + +* ``ADSENSE_SLOT_BOTTOM`` + +* ``STAFF_TOKEN``: enabled access to staff-only areas with ?staff= + +* ``FLATTR_KEY`` + +* ``FLATTR_SECRET`` + +* ``FLATTR_MYGPO_THING``: Flattr thing of the webservice. Will be flattr'd + when a user sets the "Auto-Flattr gpodder.net" option + +* ``USER_AGENT``: The User-Agent string used for outgoing HTTP requests + +* ``DEFAULT_BASE_URL``: Base URL of the website that is used if the actually + used parameters is not available. Request handlers, for example, can access + the requested domain. Code that runs in background can not do this, and + therefore requires a default value. This should be set to something like + ``http://example.com`` + +* ``BROKER_URL`` Celery Broker URL + +* ``CELERY_RESULT_BACKEND`` + +* ``CELERY_SEND_TASK_ERROR_EMAILS`` + +* ``SERVER_EMAIL`` + +* ``GOOGLE_CLIENT_ID`` + +* ``GOOGLE_CLIENT_SECRET`` + +* ``SUPPORT_URL``: URL where users of the site can get support + +* ``ELASTICSEARCH_SERVER`` + +* ``ELASTICSEARCH_INDEX`` + +* ``ELASTICSEARCH_TIMEOUT`` + +* ``ACTIVATION_VALID_DAYS`` time for how long an activation is valid; after + that, an unactivated user will be deleted + +* ``INTERNAL_IPS`` diff --git a/doc/ops/docker.rst b/doc/ops/docker.rst new file mode 100644 index 000000000..6e5fa65f5 --- /dev/null +++ b/doc/ops/docker.rst @@ -0,0 +1,57 @@ +Using Docker +============ + +mygpo can be run using `Docker `_. + + +Database +-------- + +The image requires a PostgreSQL server, specified either via + +* a `linked container `_ + called ``db`` containing a server with a database called ``mygpo``, a user + called ``mygpo`` with a password ``mygpo``. +* A ``DATABASE_URL`` environment variable (eg + ``postgres://USER:PASSWORD@HOST:PORT/NAME``) + +Using a PostgreSQL Docker container +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Start a container using the `official PostgreSQL image `_. :: + + docker run --name db -d postgres + +Create the schema and a corresponding user :: + + docker exec -it db psql -U postgres + +And enter the following commands (change passwords as required) + +.. literalinclude:: ../../contrib/init-db.sql + :language: sql + :linenos: + +Initialize the tables. This needs needs to be run for every update. :: + + sudo docker run --rm --link db:db -e SECRET_KEY=asdf mygpo/web python manage.py migrate + + +Redis +----- + + +Web Frontend +------------ + +The image exposes the web interface on port 8000. + + +Celery Worker +------------- + + +Celery Heartbeat +---------------- + + diff --git a/doc/ops/index.rst b/doc/ops/index.rst new file mode 100644 index 000000000..4b8d278fe --- /dev/null +++ b/doc/ops/index.rst @@ -0,0 +1,9 @@ +Operation Documentation +======================= + + +.. toctree:: + :maxdepth: 1 + + configuration + docker diff --git a/docker-compose-common.yml b/docker-compose-common.yml new file mode 100644 index 000000000..e84cbf4e4 --- /dev/null +++ b/docker-compose-common.yml @@ -0,0 +1,14 @@ + +version: '2' + +services: + + base: + build: + context: . + dockerfile: Dockerfile + env_file: contrib/docker.env + volumes: + - .:/srv/mygpo + - ./contrib/docker.env:/docker.env + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..31ced40cd --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,57 @@ + +version: '2' + +services: + + postgres: + build: + context: contrib + dockerfile: Dockerfile.postgres + environment: + POSTGRES_HOST_AUTH_METHOD: 'trust' + + memcached: + image: memcached:1.4 + + redis: + image: redis:3.2-alpine + + web: + extends: + file: docker-compose-common.yml + service: base + links: + - redis + - postgres + - memcached + ports: + - '8000:8000' + depends_on: + - postgres + - redis + + worker: + extends: + file: docker-compose-common.yml + service: base + command: /srv/mygpo/contrib/wait-for-postgres.py celery -A mygpo worker --concurrency=3 -l info -Ofair + links: + - web + - postgres + - redis + depends_on: + - postgres + - redis + + beat: + extends: + file: docker-compose-common.yml + service: base + command: /srv/mygpo/contrib/wait-for-postgres.py celery -A mygpo beat --pidfile /tmp/celerybeat.pid -S django + links: + - web + - postgres + - redis + depends_on: + - postgres + - redis diff --git a/makefile b/makefile index ef6a8e265..8fb5d893f 100644 --- a/makefile +++ b/makefile @@ -1,3 +1,5 @@ +APT=sudo apt-get + all: help help: @@ -34,9 +36,15 @@ clean: git clean -fX install-deps: - sudo apt-get install libpq-dev libjpeg-dev zlib1g-dev libwebp-dev \ + $(APT) install libpq-dev libjpeg-dev zlib1g-dev libwebp-dev gettext \ build-essential python3-dev virtualenv libffi-dev redis postgresql +docker-build: + sudo docker build -t="mygpo/web" . + +docker-run: + sudo docker run --rm -p 8000:8000 --name web --link db:db -e SECRET_KEY=asdf mygpo/web + format-code: black --target-version py36 --skip-string-normalization mygpo/ diff --git a/mygpo/locale/fr/LC_MESSAGES/django.po b/mygpo/locale/fr/LC_MESSAGES/django.po index e968b0862..257bc6c64 100644 --- a/mygpo/locale/fr/LC_MESSAGES/django.po +++ b/mygpo/locale/fr/LC_MESSAGES/django.po @@ -813,7 +813,7 @@ msgstr "Podcast inconnu" #: mygpo/podcasts/models.py:700 #, python-brace-format msgid "Unknown Podcast from {domain}" -msgstr "Podcast inconnu de {domaine}" +msgstr "Podcast inconnu de {domain}" #: mygpo/podcasts/templates/episode.html:85 #: mygpo/podcasts/templates/episodes.html:27 diff --git a/mygpo/settings.py b/mygpo/settings.py index 9a662e4b5..648f8b5dc 100644 --- a/mygpo/settings.py +++ b/mygpo/settings.py @@ -34,6 +34,7 @@ def get_intOrNone(name, default): DEBUG = get_bool('DEBUG', False) + ADMINS = re.findall(r'\s*([^<]+) <([^>]+)>\s*', os.getenv('ADMINS', '')) MANAGERS = ADMINS @@ -85,7 +86,7 @@ def get_intOrNone(name, default): # Static Files -STATIC_ROOT = 'staticfiles' +STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles') STATIC_URL = '/static/' STATICFILES_DIRS = (os.path.abspath(os.path.join(BASE_DIR, '..', 'static')),) @@ -137,6 +138,7 @@ def get_intOrNone(name, default): MIDDLEWARE = [ + 'whitenoise.middleware.WhiteNoiseMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', @@ -185,6 +187,7 @@ def get_intOrNone(name, default): 'mygpo.votes', ] + try: if DEBUG: import debug_toolbar @@ -208,6 +211,7 @@ def get_intOrNone(name, default): ACCOUNT_ACTIVATION_DAYS = int(os.getenv('ACCOUNT_ACTIVATION_DAYS', 7)) + AUTHENTICATION_BACKENDS = ( 'mygpo.users.backend.CaseInsensitiveModelBackend', 'mygpo.web.auth.EmailAuthenticationBackend', @@ -229,7 +233,6 @@ def get_intOrNone(name, default): CSRF_FAILURE_VIEW = 'mygpo.web.views.csrf_failure' - DEFAULT_FROM_EMAIL = os.getenv('DEFAULT_FROM_EMAIL', '') SERVER_EMAIL = os.getenv('SERVER_EMAIL', DEFAULT_FROM_EMAIL) @@ -241,12 +244,15 @@ def get_intOrNone(name, default): GOOGLE_ANALYTICS_PROPERTY_ID = os.getenv('GOOGLE_ANALYTICS_PROPERTY_ID', '') + DIRECTORY_EXCLUDED_TAGS = os.getenv('DIRECTORY_EXCLUDED_TAGS', '').split() + FLICKR_API_KEY = os.getenv('FLICKR_API_KEY', '') SOUNDCLOUD_CONSUMER_KEY = os.getenv('SOUNDCLOUD_CONSUMER_KEY', '') + MAINTENANCE = get_bool('MAINTENANCE', False) diff --git a/mygpo/users/checks.py b/mygpo/users/checks.py index c67013869..34ca07593 100644 --- a/mygpo/users/checks.py +++ b/mygpo/users/checks.py @@ -29,8 +29,10 @@ def check_case_insensitive_users(app_configs=None, **kwargs): wid = 'users.W001' errors.append(Warning(txt, id=wid)) - except OperationalError as oe: - if 'no such table: auth_user' in str(oe): + except (OperationalError, ProgrammingError) as oe: + if 'no such table: auth_user' in str( + oe + ) or 'relation "auth_user" does not exist' in str(oe): # Ignore if the table does not yet exist, eg when initally # running ``manage.py migrate`` pass diff --git a/requirements.txt b/requirements.txt index ce8a7cc08..603ef5679 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,4 +16,5 @@ redis==3.3.11 django-celery-results==1.2.0 django-celery-beat==1.5.0 requests==2.22.0 +whitenoise==5.0.1 django-db-geventpool==3.1.0