Implement Celery (#3628)

.github/workflows/backend-test.yml

@@ -25,6 +25,9 @@ jobs:
           - 5432/tcp
         # needed because the postgres container does not provide a healthcheck
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
+      redis:
+        image: redis:7.2.3
+        options: --health-cmd "redis-cli ping" --health-interval 10s --health-timeout 5s --health-retries 5
 
     steps:
       - uses: actions/checkout@v2
@@ -59,6 +62,8 @@ jobs:
           exit $STATUS
         env:
           DATABASE_URL: postgresql://postgres:postgres@localhost:${{ job.services.postgres.ports['5432'] }}/postgres
+          CELERY_BROKER_URL: redis://redis:6379/0
+          CELERY_RESULT_BACKEND: redis://redis:6379/1
           STRIPE_SECRET_API_KEY: fake-key
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
           MEDIA_FILES_STORAGE_BACKEND: django.core.files.storage.FileSystemStorage

backend/Dockerfile

@@ -67,7 +67,8 @@ COPY . ${FUNCTION_DIR}
 
 ENV DJANGO_SETTINGS_MODULE=pycon.settings.prod
 
-RUN AWS_MEDIA_BUCKET=example \
+RUN DJANGO_SETTINGS_MODULE=pycon.settings.test \
+    AWS_MEDIA_BUCKET=example \
     AWS_REGION_NAME=eu-central-1 \
     SECRET_KEY=DEMO \
     STRIPE_SECRET_API_KEY=demo \

backend/cms/components/page/signals.py

@@ -1,70 +1,22 @@
-import requests
 import logging
 
 from cms.components.sites.models import VercelFrontendSettings
+from cms.components.page.tasks import revalidate_vercel_frontend_task
 
 logger = logging.getLogger(__name__)
 
 
 def revalidate_vercel_frontend(sender, **kwargs):
     instance = kwargs["instance"]
 
-    site = kwargs["instance"].get_site()
+    site = instance.get_site()
     if not site:
         # page doesn't belong to any site
         return
 
-    site_name = site.site_name
-    hostname = site.hostname
     settings = VercelFrontendSettings.for_site(site)
 
-    if not settings:
-        # not configured for this site
+    if not settings.revalidate_url:
         return
 
-    url = settings.revalidate_url
-    secret = settings.revalidate_secret
-
-    if not url or not secret:
-        # not configured for this site
-        return
-
-    language_code = instance.locale.language_code
-
-    if language_code != "en":
-        # we need to get the original slug
-        # as we use the english slugs for the frontend
-        english_page = (
-            instance.get_translations(inclusive=True)
-            .filter(locale__language_code="en")
-            .first()
-        )
-
-        slug = english_page.slug
-        _, _, page_path = english_page.get_url_parts()
-    else:
-        slug = instance.slug
-        _, _, page_path = instance.get_url_parts()
-
-    page_path = page_path[:-1]
-
-    if slug == hostname:
-        path = f"/{language_code}"
-    else:
-        path = f"/{language_code}{page_path}"
-
-    try:
-        response = requests.post(
-            url,
-            timeout=None,
-            json={
-                "secret": secret,
-                "path": path,
-            },
-        )
-        response.raise_for_status()
-    except Exception as e:
-        logger.error(f"Error while revalidating {path} on {site_name}: {e}")
-        return
-
-    logger.info(f"Revalidated {path} on {site_name}")
+    revalidate_vercel_frontend_task.delay(page_id=instance.id)

backend/cms/components/page/tasks.py

@@ -0,0 +1,65 @@
+import requests
+import logging
+from pycon.celery import app
+from wagtail.models import Page
+from cms.components.sites.models import VercelFrontendSettings
+
+logger = logging.getLogger(__name__)
+
+
+@app.task
+def revalidate_vercel_frontend_task(page_id):
+    page = Page.objects.get(id=page_id)
+    site = page.get_site()
+
+    settings = VercelFrontendSettings.for_site(site)
+
+    site_name = site.site_name
+    hostname = site.hostname
+
+    url = settings.revalidate_url
+    secret = settings.revalidate_secret
+
+    if not url or not secret:
+        # not configured for this site
+        return
+
+    language_code = page.locale.language_code
+
+    if language_code != "en":
+        # we need to get the original slug
+        # as we use the english slugs for the frontend
+        english_page = (
+            page.get_translations(inclusive=True)
+            .filter(locale__language_code="en")
+            .first()
+        )
+
+        slug = english_page.slug
+        _, _, page_path = english_page.get_url_parts()
+    else:
+        slug = page.slug
+        _, _, page_path = page.get_url_parts()
+
+    page_path = page_path[:-1]
+
+    if slug == hostname:
+        path = f"/{language_code}"
+    else:
+        path = f"/{language_code}{page_path}"
+
+    try:
+        response = requests.post(
+            url,
+            timeout=None,
+            json={
+                "secret": secret,
+                "path": path,
+            },
+        )
+        response.raise_for_status()
+    except Exception as e:
+        logger.error(f"Error while revalidating {path} on {site_name}: {e}")
+        return
+
+    logger.info(f"Revalidated {path} on {site_name}")

backend/cms/components/page/tests/test_signals.py

@@ -1,119 +1,35 @@
+from cms.components.sites.tests.factories import VercelFrontendSettingsFactory
+from unittest import mock
 import pytest
 from cms.components.page.signals import revalidate_vercel_frontend
-from cms.components.sites.tests.factories import VercelFrontendSettingsFactory
 from wagtail_factories import PageFactory, SiteFactory
 
 pytestmark = pytest.mark.django_db
 
 
-def test_revalidate_vercel_frontend_disabled_if_not_configured(requests_mock):
-    mock_call = requests_mock.post("https://test.com", status_code=200)
-
+@mock.patch("cms.components.page.signals.revalidate_vercel_frontend_task")
+def test_revalidate_vercel_frontend_disabled_if_not_configured(mock_task):
     site = SiteFactory()
     page = PageFactory()
     site.root_page = page
     site.save()
-    revalidate_vercel_frontend("test_revalidate_vercel_frontend", instance=page)
-
-    assert not mock_call.called
-
-
-def test_revalidate_vercel_frontend(
-    requests_mock,
-):
-    site = SiteFactory()
-    parent = PageFactory()
-    page = PageFactory(slug="test-page123")
-    page.set_url_path(parent)
-
-    site.root_page = parent
-    site.save()
-
-    settings = VercelFrontendSettingsFactory(
-        revalidate_url="https://test.com", revalidate_secret="test", site=site
-    )
-    mock_call = requests_mock.post(settings.revalidate_url, status_code=200)
 
     revalidate_vercel_frontend("test_revalidate_vercel_frontend", instance=page)
 
-    assert mock_call.called
-    body = mock_call.last_request.json()
-    assert body["secret"] == "test"
-    assert body["path"] == "/en/test-page123"
+    mock_task.delay.assert_not_called()
 
 
-def test_revalidate_vercel_frontend_special_case_for_landing_page(
-    requests_mock,
-):
+@mock.patch("cms.components.page.signals.revalidate_vercel_frontend_task")
+def test_revalidate_vercel_frontend(mock_task):
     site = SiteFactory()
-
-    parent = PageFactory()
-    page = PageFactory(slug=site.hostname)
-    page.set_url_path(parent)
-    site.root_page = parent
-
+    page = PageFactory()
+    site.root_page = page
     site.save()
 
-    settings = VercelFrontendSettingsFactory(
+    VercelFrontendSettingsFactory(
         revalidate_url="https://test.com", revalidate_secret="test", site=site
     )
-    mock_call = requests_mock.post(settings.revalidate_url, status_code=200)
 
     revalidate_vercel_frontend("test_revalidate_vercel_frontend", instance=page)
 
-    assert mock_call.called
-
-    body = mock_call.last_request.json()
-    assert body["secret"] == "test"
-    assert body["path"] == "/en"
-
-
-def test_revalidate_vercel_frontend_for_different_language(requests_mock, locale):
-    parent = PageFactory()
-    site = SiteFactory(hostname="pycon", root_page=parent)
-
-    page = PageFactory(slug="test123", locale=locale("en"), parent=parent)
-    page.set_url_path(parent)
-
-    italian_page = page.copy_for_translation(locale=locale("it"))
-    italian_page.slug = "something-else"
-    italian_page.save()
-    italian_page.set_url_path(parent)
-
-    settings = VercelFrontendSettingsFactory(
-        revalidate_url="https://test.com", revalidate_secret="test", site=site
-    )
-    mock_call = requests_mock.post(settings.revalidate_url, status_code=200)
-
-    revalidate_vercel_frontend("test_revalidate_vercel_frontend", instance=italian_page)
-
-    assert mock_call.called
-
-    body = mock_call.last_request.json()
-    assert body["secret"] == "test"
-    assert body["path"] == "/it/test123"
-
-
-def test_revalidate_vercel_frontend_when_vercel_is_down_doesnt_crash(
-    caplog,
-    requests_mock,
-    locale,
-):
-    parent = PageFactory()
-
-    page = PageFactory(slug="test123", locale=locale("en"), parent=parent)
-    site = SiteFactory(hostname="pycon", root_page=page)
-
-    italian_page = page.copy_for_translation(locale=locale("it"))
-    italian_page.slug = "something-else"
-    italian_page.save()
-
-    settings = VercelFrontendSettingsFactory(
-        revalidate_url="https://test.com", revalidate_secret="test", site=site
-    )
-    mock_call = requests_mock.post(settings.revalidate_url, status_code=500)
-
-    revalidate_vercel_frontend("test_revalidate_vercel_frontend", instance=italian_page)
-
-    assert mock_call.called
-    assert "Error while revalidating" in caplog.records[0].message
+    mock_task.delay.assert_called_with(page_id=page.id)