ISG for i18n js catalogs

Dockerfile

@@ -91,6 +91,11 @@ rm -rf /deps/build/*
 ${PIP_COMMAND} install --progress-bar=off --no-deps --exists-action=w -r requirements/pip.txt
 EOF
 
+# Expose the DOCKER_TARGET variable to all subsequent stages
+# This value is used to determine if we are building for production or development
+ARG DOCKER_TARGET
+ENV DOCKER_TARGET=${DOCKER_TARGET}
+
 # Define production dependencies as a single layer
 # let's the rest of the stages inherit prod dependencies
 # and makes copying the /deps dir to the final layer easy.
@@ -175,9 +180,6 @@ ENV DOCKER_VERSION=${DOCKER_VERSION}
 COPY docker/etc/mime.types /etc/mime.types
 # Copy the rest of the source files from the host
 COPY --chown=olympia:olympia . ${HOME}
-# Copy assets from assets
-COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-static
-COPY --from=assets --chown=olympia:olympia ${HOME}/static-build ${HOME}/static-build
 
 # Set shell back to sh until we can prove we can use bash at runtime
 SHELL ["/bin/sh", "-c"]
@@ -191,6 +193,9 @@ FROM sources AS production
 
 # Copy compiled locales from builder
 COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
+# Copy assets from assets
+COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-static
+COPY --from=assets --chown=olympia:olympia ${HOME}/static-build ${HOME}/static-build
 # Copy dependencies from `pip_production`
 COPY --from=pip_production --chown=olympia:olympia /deps /deps
 

Makefile-docker

@@ -28,7 +28,12 @@ check_debian_packages: ## check the existence of multiple debian packages
 
 .PHONY: check_pip_packages
 check_pip_packages: ## check the existence of multiple python packages
-	./scripts/check_pip_packages.sh prod.txt dev.txt
+	@ ./scripts/check_pip_packages.sh prod.txt
+# "development" corresponds to the "development" DOCKER_TARGET defined in the Dockerfile
+# When the target is "development" it means we expect dev.txt dependencies to be installed.
+	@if [ "$(DOCKER_TARGET)" = "development" ]; then \
+		./scripts/check_pip_packages.sh dev.txt; \
+	fi
 
 .PHONY: check_files
 check_files: ## check the existence of multiple files

docker-bake.hcl

@@ -18,6 +18,7 @@ target "web" {
 	DOCKER_COMMIT = "${DOCKER_COMMIT}"
 	DOCKER_VERSION = "${DOCKER_VERSION}"
 	DOCKER_BUILD = "${DOCKER_BUILD}"
+  DOCKER_TARGET = "${DOCKER_TARGET}"
   }
   pull = true
 

docker-compose.ci.yml

@@ -2,7 +2,6 @@ services:
   worker:
     environment:
       - HOST_UID=9500
-      - DEBUG=
     volumes:
       - /data/olympia
 

docker-compose.yml

@@ -52,10 +52,6 @@ services:
     ]
     volumes:
       - data_olympia:/data/olympia
-      # Don't mount generated files. They only exist in the container
-      # and would otherwiser be deleted by mounbting data_olympia
-      - /data/olympia/static-build
-      - /data/olympia/site-static
       - storage:/data/olympia/storage
       - ./package.json:/deps/package.json
       - ./package-lock.json:/deps/package-lock.json

docs/topics/development/static-files.md

@@ -59,23 +59,3 @@ During development they are served by the django development server.
 
 We have a (complex) set of npm static assets that are built by the `compress_assets` management command.
 During development, these assets are served directly from the node_modules directory using a custom static finder.
-
-## DEBUG Property and Static File Serving
-
-The behavior of static file serving can be controlled using the `DEBUG` environment variable or via setting it directly in
-the `local_settings.py` file. Be careful directly setting this value, if DEBUG is set to false, and you don't have sufficient
-routing setup to serve files fron nginx only, it can cause failure to serve some static files.
-
-It is best to use  the compose file to control DEBUG.a
-
-This is set in the environment, and in CI environments, it's controlled by the `docker-compose.ci.yml` file.
-
-The `DEBUG` property is what is used by django to determine if it should serve static files or not. In development,
-you can manually override this in the make up command, but in general, you should rely on the `docker-compose.ci.yml` file
-to set the correct value as this will also set appropriate file mounts.
-
-```bash
-make up COMPOSE_FILE=docker-compose.yml:docker-compose.ci.yml
-```
-
-This will run addons-server in production mode, serving files from the `site-static` directory.

docs/topics/development/troubleshooting_and_debugging.md

@@ -2,6 +2,22 @@
 
 Effective troubleshooting and debugging practices are essential for maintaining and improving the **addons-server** project. This section covers common issues, their solutions, and tools for effective debugging.
 
+## DEV_MODE vs DEBUG
+
+In our project, `DEV_MODE` and `DEBUG` serve distinct but complementary purposes.
+`DEV_MODE` is directly tied to the `DOCKER_TARGET` environment variable and is used to enable or disable behaviors
+based on whether we are running a production image or not.
+
+For instance, production images always disables certain features like using fake fxa authentication. Additionally,
+certain dependencies are only installed in [dev.txt](../../../requirements/dev.txt) and so must be disabled in production.
+
+Conversely, `DEBUG` controls the activation of debugging tools and utilities, such as the debug_toolbar,
+which are useful for troubleshooting. Unlike DEV_MODE, DEBUG is independent
+and can be toggled in both development and production environments as needed.
+
+This separation ensures that essential behaviors are managed according to the deployment target (DEV_MODE),
+while allowing flexibility to enable or disable debugging features (DEBUG) in production or development images.
+
 ## Common Issues and Solutions
 
 1. **Containers Not Starting**:

settings.py

@@ -17,10 +17,7 @@
 INTERNAL_ROUTES_ALLOWED = True
 
 # These apps are great during development.
-INSTALLED_APPS += (
-    'olympia.landfill',
-    'dbbackup',
-)
+INSTALLED_APPS += ('olympia.landfill',)
 
 DBBACKUP_STORAGE = 'django.core.files.storage.FileSystemStorage'
 
@@ -52,8 +49,11 @@ def insert_debug_toolbar_middleware(middlewares):
     return tuple(ret_middleware)
 
 
-if DEBUG:
-    INSTALLED_APPS += ('debug_toolbar',)
+if DEV_MODE:
+    INSTALLED_APPS += (
+        'debug_toolbar',
+        'dbbackup',
+    )
     MIDDLEWARE = insert_debug_toolbar_middleware(MIDDLEWARE)
 
 DEBUG_TOOLBAR_CONFIG = {
@@ -106,7 +106,7 @@ def insert_debug_toolbar_middleware(middlewares):
 FXA_OAUTH_HOST = 'https://oauth.stage.mozaws.net/v1'
 FXA_PROFILE_HOST = 'https://profile.stage.mozaws.net/v1'
 
-# When USE_FAKE_FXA_AUTH and settings.DEBUG are both True, we serve a fake
+# When USE_FAKE_FXA_AUTH and settings.DEV_MODE are both True, we serve a fake
 # authentication page, bypassing FxA. To disable this behavior, set
 # USE_FAKE_FXA = False in your local settings.
 # You will also need to specify `client_id` and `client_secret` in your

settings_test.py

@@ -23,6 +23,8 @@
 IN_TEST_SUITE = True
 
 DEBUG = False
+# We should default to production mode unless otherwiser specified
+DEV_MODE = False
 
 # We won't actually send an email.
 SEND_REAL_EMAIL = True

src/olympia/accounts/tests/test_utils.py

@@ -292,7 +292,7 @@ def test_redirect_for_login_with_2fa_enforced_and_config():
     assert request.session['enforce_2fa'] is True
 
 
-@override_settings(DEBUG=True, USE_FAKE_FXA_AUTH=True)
+@override_settings(DEV_MODE=True, USE_FAKE_FXA_AUTH=True)
 def test_fxa_login_url_when_faking_fxa_auth():
     path = '/en-US/addons/abp/?source=ddg'
     request = RequestFactory().get(path)

src/olympia/accounts/tests/test_verify.py

@@ -234,7 +234,7 @@ def test_with_id_token(self):
         self.get_profile.assert_called_with('cafe')
 
 
-@override_settings(USE_FAKE_FXA_AUTH=False, DEBUG=True, VERIFY_FXA_ACCESS_TOKEN=True)
+@override_settings(USE_FAKE_FXA_AUTH=False, DEV_MODE=True, VERIFY_FXA_ACCESS_TOKEN=True)
 class TestCheckAndUpdateFxaAccessToken(TestCase):
     def setUp(self):
         super().setUp()

src/olympia/accounts/tests/test_views.py

@@ -172,7 +172,7 @@ def has_cors_headers(response, origin='https://addons-frontend'):
 
 
 class TestLoginStartView(TestCase):
-    @override_settings(DEBUG=True, USE_FAKE_FXA_AUTH=True)
+    @override_settings(DEV_MODE=True, USE_FAKE_FXA_AUTH=True)
     def test_redirect_url_fake_fxa_auth(self):
         response = self.client.get(reverse_ns('accounts.login_start'))
         assert response.status_code == 302
@@ -700,7 +700,7 @@ def test_waffle_flag_off_enforced_2fa_should_have_no_effect(self):
         self.request.session['enforce_2fa'] = True
         self._test_should_continue_without_redirect_for_two_factor_auth()
 
-    @override_settings(DEBUG=True, USE_FAKE_FXA_AUTH=True)
+    @override_settings(DEV_MODE=True, USE_FAKE_FXA_AUTH=True)
     def test_fake_fxa_auth(self):
         self.user = user_factory()
         self.find_user.return_value = self.user
@@ -721,7 +721,7 @@ def test_fake_fxa_auth(self):
         assert kwargs['next_path'] == '/a/path/?'
         assert self.fxa_identify.call_count == 0
 
-    @override_settings(DEBUG=True, USE_FAKE_FXA_AUTH=True)
+    @override_settings(DEV_MODE=True, USE_FAKE_FXA_AUTH=True)
     def test_fake_fxa_auth_with_2fa(self):
         self.user = user_factory()
         self.find_user.return_value = self.user

src/olympia/amo/tests/test_views.py

@@ -510,7 +510,7 @@ def test_allow_mozilla_collections(self):
 
 @pytest.mark.django_db
 def test_fake_fxa_authorization_correct_values_passed():
-    with override_settings(DEBUG=True):  # USE_FAKE_FXA_AUTH is already True
+    with override_settings(DEV_MODE=True):  # USE_FAKE_FXA_AUTH is already True
         url = reverse('fake-fxa-authorization')
         response = test.Client().get(url, {'state': 'foobar'})
         assert response.status_code == 200
@@ -528,15 +528,15 @@ def test_fake_fxa_authorization_correct_values_passed():
 @pytest.mark.django_db
 def test_fake_fxa_authorization_deactivated():
     url = reverse('fake-fxa-authorization')
-    with override_settings(DEBUG=False, USE_FAKE_FXA_AUTH=False):
+    with override_settings(DEV_MODE=False, USE_FAKE_FXA_AUTH=False):
         response = test.Client().get(url)
     assert response.status_code == 404
 
-    with override_settings(DEBUG=False, USE_FAKE_FXA_AUTH=True):
+    with override_settings(DEV_MODE=False, USE_FAKE_FXA_AUTH=True):
         response = test.Client().get(url)
     assert response.status_code == 404
 
-    with override_settings(DEBUG=True, USE_FAKE_FXA_AUTH=False):
+    with override_settings(DEV_MODE=True, USE_FAKE_FXA_AUTH=False):
         response = test.Client().get(url)
     assert response.status_code == 404
 

src/olympia/amo/utils.py

@@ -1165,7 +1165,7 @@ def extract_colors_from_image(path):
 def use_fake_fxa():
     """Return whether or not to use a fake FxA server for authentication.
     Should always return False in production"""
-    return settings.DEBUG and settings.USE_FAKE_FXA_AUTH
+    return settings.DEV_MODE and settings.USE_FAKE_FXA_AUTH
 
 
 class AMOJSONEncoder(JSONEncoder):

src/olympia/api/urls.py

@@ -27,7 +27,7 @@ def get_versioned_api_routes(version, url_patterns):
     routes = url_patterns
 
     # For now, this feature is only enabled in dev mode
-    if settings.DEBUG:
+    if settings.DEV_MODE:
         routes.extend(
             [
                 re_path(

src/olympia/core/apps.py

@@ -63,6 +63,9 @@ def static_check(app_configs, **kwargs):
     errors = []
     output = StringIO()
 
+    if settings.DEV_MODE:
+        return []
+
     try:
         call_command('compress_assets', dry_run=True, stdout=output)
         file_paths = output.getvalue().strip().split('\n')

src/olympia/landfill/management/commands/fetch_prod_addons.py

@@ -43,9 +43,9 @@ def add_arguments(self, parser):
         )
 
     def handle(self, *args, **options):
-        if not settings.DEBUG:
+        if not settings.DEV_MODE:
             raise CommandError(
-                'As a safety precaution this command only works if DEBUG=True.'
+                'As a safety precaution this command only works in DEV_MODE.'
             )
         self.fetch_addon_data(options)
 

src/olympia/landfill/management/commands/fetch_prod_versions.py

@@ -28,9 +28,9 @@ def add_arguments(self, parser):
         )
 
     def handle(self, *args, **options):
-        if not settings.DEBUG:
+        if not settings.DEV_MODE:
             raise CommandError(
-                'As a safety precaution this command only works if DEBUG=True.'
+                'As a safety precaution this command only works in DEV_MODE.'
             )
         self.options = options
         self.fetch_versions_data()

src/olympia/landfill/management/commands/generate_addons.py

@@ -46,10 +46,8 @@ def add_arguments(self, parser):
         )
 
     def handle(self, *args, **kwargs):
-        if not settings.DEBUG:
-            raise CommandError(
-                'You can only run this command with your DEBUG setting set to True.'
-            )
+        if not settings.DEV_MODE:
+            raise CommandError('You can only run this command in DEV_MODE.')
 
         num = int(kwargs.get('num'))
         email = kwargs.get('email')

src/olympia/landfill/management/commands/generate_themes.py

@@ -37,10 +37,8 @@ def add_arguments(self, parser):
         )
 
     def handle(self, *args, **kwargs):
-        if not settings.DEBUG:
-            raise CommandError(
-                'You can only run this command with your DEBUG setting set to True.'
-            )
+        if not settings.DEV_MODE:
+            raise CommandError('You can only run this command in DEV_MODE.')
         num = int(kwargs.get('num'))
         email = kwargs.get('email')
 

src/olympia/lib/settings_base.py

@@ -59,6 +59,17 @@ def path(*folders):
 
 DEBUG = env('DEBUG', default=False)
 
+# Do NOT provide a default value, this should be explicitly
+# set during the docker image build. If it is not set,
+# we want to raise an error.
+DOCKER_TARGET = env('DOCKER_TARGET')
+
+# "production" is a named docker stage corresponding to the production image.
+# when we build the production image, the stage to use is determined
+# via the "DOCKER_TARGET" variable which is also passed into the image.
+# So if the value is anything other than "production" we are in development mode.
+DEV_MODE = DOCKER_TARGET != 'production'
+
 DEBUG_TOOLBAR_CONFIG = {
     # Deactivate django debug toolbar by default.
     'SHOW_TOOLBAR_CALLBACK': lambda request: DEBUG,

src/olympia/urls.py

@@ -2,6 +2,7 @@
 from django.contrib import admin
 from django.shortcuts import redirect
 from django.urls import include, re_path, reverse
+from django.views.i18n import JavaScriptCatalog
 from django.views.static import serve as serve_static
 
 from olympia.amo.utils import urlparams
@@ -109,7 +110,7 @@
     ),
 ]
 
-if settings.DEBUG:
+if settings.DEV_MODE:
     from django.contrib.staticfiles.views import serve as static_serve
 
     def serve_static_files(request, path, **kwargs):
@@ -125,6 +126,13 @@ def serve_static_files(request, path, **kwargs):
                 serve_static,
                 {'document_root': settings.MEDIA_ROOT},
             ),
+            # Specific fallback for statically generated i18n js files
+            # Not generated in dev mode builds
+            re_path(
+                r'^static/js/i18n/.*\.js$',
+                JavaScriptCatalog.as_view(),
+                name='js-i18n-catalog',
+            ),
             # fallback for static files that are not available directly over nginx.
             # Mostly vendor files from python or npm dependencies that are not available
             # in the static files directory.