Add build_info.py including content hash verification for static buil…

…d directories:

- Removed the previous build information generation method and replaced it with a new Python script (scripts/build_info.py) that creates a build info file with content hashes for static assets and locales.
- Updated Dockerfile to utilize the new build_info.py script for generating build information during the Docker build process.
- Adjusted the static check in the Django app to validate the content hash of static files against the expected hash.
- Cleaned up .dockerignore and .gitignore files by removing the exclusion of build*.py files.
- Added unit tests for the new build_info.py script and its functionality.

.dockerignore

@@ -26,7 +26,6 @@
 .tox/
 .vscode
 backups
-build*.py
 buildx-bake-metadata.json
 deps/*
 docker*.yml

.gitignore

@@ -25,7 +25,6 @@
 .tox/
 .vscode
 backups
-build*.py
 buildx-bake-metadata.json
 deps/*
 docker*.yml

Dockerfile

@@ -6,6 +6,12 @@ FROM python:3.12-slim-bookworm AS olympia
 
 ENV BUILD_INFO=/build-info.json
 
+# Set permissions to make the file readable by all but only writable by root
+RUN <<EOF
+echo "{}" > ${BUILD_INFO}
+chmod 644 ${BUILD_INFO}
+EOF
+
 # Set shell to bash with logs and errors for build
 SHELL ["/bin/bash", "-xue", "-c"]
 
@@ -20,29 +26,14 @@ ENV HOME=/data/olympia
 WORKDIR ${HOME}
 RUN chown -R olympia:olympia ${HOME}
 
-FROM olympia AS info
+FROM olympia AS build_info
 
-# Build args that represent static build information
-# These are passed to docker via the bake.hcl file and
-# should not be overridden in the container environment.
-ARG DOCKER_COMMIT
-ARG DOCKER_VERSION
-ARG DOCKER_BUILD
+# We only need the DOCKER_TARGET at this point
 ARG DOCKER_TARGET
-
 # Create the build file hard coding build variables to the image
-RUN <<EOF
-cat <<INNEREOF > ${BUILD_INFO}
-{
-  "commit": "${DOCKER_COMMIT}",
-  "version": "${DOCKER_VERSION}",
-  "build": "${DOCKER_BUILD}",
-  "target": "${DOCKER_TARGET}",
-  "source": "https://github.com/mozilla/addons-server"
-}
-INNEREOF
-# Set permissions to make the file readable by all but only writable by root
-chmod 644 ${BUILD_INFO}
+RUN --mount=type=bind,source=scripts/build_info.py,target=${HOME}/scripts/build_info.py \
+<<EOF
+${HOME}/scripts/build_info.py --output ${BUILD_INFO} --target ${DOCKER_TARGET}
 EOF
 
 FROM olympia AS base
@@ -147,7 +138,7 @@ EOF
 FROM base AS development
 
 # Copy build info from info
-COPY --from=info ${BUILD_INFO} ${BUILD_INFO}
+COPY --from=build_info ${BUILD_INFO} ${BUILD_INFO}
 
 FROM base AS locales
 ARG LOCALE_DIR=${HOME}/locale
@@ -178,12 +169,42 @@ COPY --chown=olympia:olympia static/ ${HOME}/static/
 RUN \
     --mount=type=bind,src=src,target=${HOME}/src \
     --mount=type=bind,src=Makefile-docker,target=${HOME}/Makefile-docker \
+    --mount=type=bind,src=scripts/build_info.py,target=${HOME}/scripts/build_info.py \
     --mount=type=bind,src=scripts/update_assets.py,target=${HOME}/scripts/update_assets.py \
     --mount=type=bind,src=manage.py,target=${HOME}/manage.py \
 <<EOF
 make -f Makefile-docker update_assets
 EOF
 
+# This stage extends from olympia so we can run as root
+# prod_info includes production build directories to hash content
+# ensuring built assets are not modified at runtime.
+FROM olympia AS prod_info
+
+# Copy compiled locales from builder
+COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
+# Copy assets from assets
+COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-static
+
+# Build args that represent static build information
+# These are passed to docker via the bake.hcl file and
+# should not be set in the container environment.
+ARG DOCKER_COMMIT
+ARG DOCKER_VERSION
+ARG DOCKER_BUILD
+ARG DOCKER_TARGET
+
+RUN \
+    --mount=type=bind,source=scripts/build_info.py,target=${HOME}/scripts/build_info.py \
+<<EOF
+${HOME}/scripts/build_info.py \
+  --output "${BUILD_INFO}" \
+  --commit "${DOCKER_COMMIT}" \
+  --version "${DOCKER_VERSION}" \
+  --build "${DOCKER_BUILD}" \
+  --target "${DOCKER_TARGET}"
+EOF
+
 FROM base AS production
 # Copy the rest of the source files from the host
 COPY --chown=olympia:olympia . ${HOME}
@@ -193,7 +214,7 @@ COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-static
 COPY --from=assets --chown=olympia:olympia ${HOME}/static-build ${HOME}/static-build
 # Copy build info from info
-COPY --from=info ${BUILD_INFO} ${BUILD_INFO}
+COPY --from=prod_info ${BUILD_INFO} ${BUILD_INFO}
 # Copy compiled locales from builder
 COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 # Copy dependencies from `pip_production`

scripts/build_info.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import hashlib
+from pathlib import Path
+
+root = Path(__file__).parent.parent
+
+def hash_directory(
+  directory: Path, exclude_patterns: list[str] = None, verbose: bool = False,
+) -> str:
+    if not directory.exists():
+        return None
+
+    hasher = hashlib.blake2b(digest_size=32)
+
+    exclude_patterns = exclude_patterns or []
+
+    files = sorted(f for f in directory.rglob('*'))
+
+    for f in files:
+        is_hidden = any(part.startswith('.') for part in f.parts)
+        is_excluded = any(f.match(pattern) for pattern in exclude_patterns)
+        if not is_hidden and not is_excluded:
+            if verbose:
+                print(f'Hashing {f}')
+            hasher.update(f.name.encode())
+            if f.is_file():
+                hasher.update(f.read_bytes())
+        elif verbose:
+            print(f'Skipping {f}')
+
+    return hasher.hexdigest()
+
+
+def build_info(commit: str = None, version: str = None, build: str = None, target: str = None):
+  """
+  Create a build info file with the current build information from the environment.
+  """
+  return {
+    "commit": commit,
+    "version": version,
+    "build": build,
+    "target": target,
+    "source": "https://github.com/mozilla/addons-server",
+    "content_hash": {
+      "site_static_hash": hash_directory((root / 'site-static'), exclude_patterns=['staticfiles.json']),
+      "locale_hash": hash_directory((root /'locale'))
+    }
+  }
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--output", type=str, required=False)
+
+  parser.add_argument("--commit", type=str, required=False)
+  parser.add_argument("--version", type=str, required=False)
+  parser.add_argument("--build", type=str, required=False)
+  # Docker target is used to determine build time and runtime behavior
+  parser.add_argument("--target", type=str, required=True)
+
+  args = parser.parse_args()
+
+  version = build_info(args.commit, args.version, args.build, args.target)
+
+  if args.output:
+    with open(args.output, 'w') as f:
+      json.dump(version, f)
+  else:
+    print(json.dumps(version))
+

src/olympia/core/apps.py

@@ -1,22 +1,21 @@
+import json
 import logging
 import os
 import subprocess
 import warnings
-from io import StringIO
 from pwd import getpwnam
 
 from django.apps import AppConfig
 from django.conf import settings
 from django.core.checks import Error, Tags, register
-from django.core.management import call_command
-from django.core.management.base import CommandError
 from django.db import connection
 from django.utils.translation import gettext_lazy as _
 
 import requests
 
 from olympia.core.utils import REQUIRED_VERSION_KEYS, get_version_json
 
+from scripts.build_info import build_info
 
 log = logging.getLogger('z.startup')
 
@@ -85,46 +84,25 @@ def version_check(app_configs, **kwargs):
 
 @register(CustomTags.custom_setup)
 def static_check(app_configs, **kwargs):
-    errors = []
-    output = StringIO()
+    """
+    Check that the static files have not been modified since the original build.
+    """
     version = get_version_json()
 
     # We only run this check in production images.
     if version.get('target') != 'production':
         return []
 
-    try:
-        call_command('compress_assets', dry_run=True, stdout=output)
-        stripped_output = output.getvalue().strip()
-
-        if stripped_output:
-            file_paths = stripped_output.split('\n')
-            for file_path in file_paths:
-                if not os.path.exists(file_path):
-                    error = f'Compressed asset file does not exist: {file_path}'
-                    errors.append(
-                        Error(
-                            error,
-                            id='setup.E003',
-                        )
-                    )
-        else:
-            errors.append(
-                Error(
-                    'No compressed asset files were found.',
-                    id='setup.E003',
-                )
-            )
-
-    except CommandError as e:
-        errors.append(
+    site_static_hash = build_info().get('content_hash').get('site_static_hash')
+    if site_static_hash != version.get('content_hash').get('site_static_hash'):
+        return [
             Error(
-                f'Error running compress_assets command: {str(e)}',
-                id='setup.E004',
+                'Site static directory does not match expected content hash',
+                id='setup.E003',
             )
-        )
+        ]
 
-    return errors
+    return []
 
 
 @register(CustomTags.custom_setup)

src/olympia/core/tests/test_apps.py

@@ -21,6 +21,10 @@ def setUp(self):
             'version': '1.0',
             'build': 'http://example.com/build',
             'source': 'https://github.com/mozilla/addons-server',
+            'content_hash': {
+                'site_static_hash': 'abc',
+                'locale_hash': 'def',
+            }
         }
         patch = mock.patch(
             'olympia.core.apps.get_version_json',
@@ -104,64 +108,6 @@ def test_illegal_override_uid_check(self, mock_getpwnam):
         with override_settings(HOST_UID=1000):
             call_command('check')
 
-    def test_static_check_no_assets_found(self):
-        """
-        Test static_check fails if compress_assets reports no files.
-        """
-        self.mock_get_version_json.return_value['target'] = 'production'
-        # Simulate "compress_assets" returning no file paths.
-        self.mock_call_command.side_effect = (
-            lambda command, dry_run, stdout: stdout.write('')
-        )
-        with self.assertRaisesMessage(
-            SystemCheckError, 'No compressed asset files were found.'
-        ):
-            call_command('check')
-
-    @mock.patch('os.path.exists')
-    def test_static_check_missing_assets(self, mock_exists):
-        """
-        Test static_check fails if at least one specified compressed
-        asset file does not exist.
-        """
-        self.mock_get_version_json.return_value['target'] = 'production'
-        # Simulate "compress_assets" returning a couple of files.
-        self.mock_call_command.side_effect = (
-            lambda command, dry_run, stdout: stdout.write(
-                f'{self.fake_css_file}\nfoo.js\n'
-            )
-        )
-        # Pretend neither file exists on disk.
-        mock_exists.return_value = False
-
-        with self.assertRaisesMessage(
-            SystemCheckError,
-            # Only the first missing file triggers the AssertionError message check
-            'Compressed asset file does not exist: foo.js',
-        ):
-            call_command('check')
-
-    def test_static_check_command_error(self):
-        """
-        Test static_check fails if there's an error during compress_assets.
-        """
-        self.mock_get_version_json.return_value['target'] = 'production'
-        self.mock_call_command.side_effect = CommandError('Oops')
-        with self.assertRaisesMessage(
-            SystemCheckError, 'Error running compress_assets command: Oops'
-        ):
-            call_command('check')
-
-    def test_static_check_command_success(self):
-        """
-        Test static_check succeeds if compress_assets runs without errors.
-        """
-        self.mock_get_version_json.return_value['target'] = 'production'
-        self.mock_call_command.side_effect = (
-            lambda command, dry_run, stdout: stdout.write(f'{self.fake_css_file}\n')
-        )
-        call_command('check')
-
     def test_nginx_skips_check_on_production_target(self):
         fake_media_root = '/fake/not/real'
         with override_settings(MEDIA_ROOT=fake_media_root):
@@ -215,3 +161,16 @@ def test_nginx_raises_unexpected_content(self):
     def test_nginx_raises_unexpected_served_by(self):
         """Test that files are served by nginx and not redirected elsewhere."""
         self._test_nginx_response('http://nginx/user-media', served_by='wow')
+
+    @mock.patch('olympia.core.apps.build_info')
+    def test_static_check(self, mock_build_info):
+        mock_build_info.return_value = {
+            'content_hash': {
+                'site_static_hash': 'abc',
+            },
+        }
+        with self.assertRaisesMessage(
+            SystemCheckError,
+            'Site static directory does not match expected content hash',
+        ):
+            call_command('check')