Add build_info.py including content hash verification for static buil…

…d directories:

- Removed the previous build information generation method and replaced it with a new Python script (scripts/build_info.py) that creates a build info file with content hashes for static assets and locales.
- Updated Dockerfile to utilize the new build_info.py script for generating build information during the Docker build process.
- Adjusted the static check in the Django app to validate the content hash of static files against the expected hash.
- Cleaned up .dockerignore and .gitignore files by removing the exclusion of build*.py files.
- Added unit tests for the new build_info.py script and its functionality.

.dockerignore

@@ -26,7 +26,6 @@
 .tox/
 .vscode
 backups
-build*.py
 buildx-bake-metadata.json
 deps/*
 docker*.yml

.gitignore

@@ -25,7 +25,6 @@
 .tox/
 .vscode
 backups
-build*.py
 buildx-bake-metadata.json
 deps/*
 docker*.yml

Dockerfile

@@ -6,6 +6,12 @@ FROM python:3.12-slim-bookworm AS olympia
 
 ENV BUILD_INFO=/build-info.json
 
+# Set permissions to make the file readable by all but only writable by root
+RUN <<EOF
+echo "{}" > ${BUILD_INFO}
+chmod 644 ${BUILD_INFO}
+EOF
+
 # Set shell to bash with logs and errors for build
 SHELL ["/bin/bash", "-xue", "-c"]
 
@@ -20,29 +26,14 @@ ENV HOME=/data/olympia
 WORKDIR ${HOME}
 RUN chown -R olympia:olympia ${HOME}
 
-FROM olympia AS info
+FROM olympia AS build_info
 
-# Build args that represent static build information
-# These are passed to docker via the bake.hcl file and
-# should not be overridden in the container environment.
-ARG DOCKER_COMMIT
-ARG DOCKER_VERSION
-ARG DOCKER_BUILD
+# We only need the DOCKER_TARGET at this point
 ARG DOCKER_TARGET
-
 # Create the build file hard coding build variables to the image
-RUN <<EOF
-cat <<INNEREOF > ${BUILD_INFO}
-{
-  "commit": "${DOCKER_COMMIT}",
-  "version": "${DOCKER_VERSION}",
-  "build": "${DOCKER_BUILD}",
-  "target": "${DOCKER_TARGET}",
-  "source": "https://github.com/mozilla/addons-server"
-}
-INNEREOF
-# Set permissions to make the file readable by all but only writable by root
-chmod 644 ${BUILD_INFO}
+RUN --mount=type=bind,source=scripts/build_info.py,target=${HOME}/scripts/build_info.py \
+<<EOF
+${HOME}/scripts/build_info.py --output ${BUILD_INFO} --target ${DOCKER_TARGET}
 EOF
 
 FROM olympia AS base
@@ -147,7 +138,7 @@ EOF
 FROM base AS development
 
 # Copy build info from info
-COPY --from=info ${BUILD_INFO} ${BUILD_INFO}
+COPY --from=build_info ${BUILD_INFO} ${BUILD_INFO}
 
 FROM base AS locales
 ARG LOCALE_DIR=${HOME}/locale
@@ -178,12 +169,42 @@ COPY --chown=olympia:olympia static/ ${HOME}/static/
 RUN \
     --mount=type=bind,src=src,target=${HOME}/src \
     --mount=type=bind,src=Makefile-docker,target=${HOME}/Makefile-docker \
+    --mount=type=bind,src=scripts/build_info.py,target=${HOME}/scripts/build_info.py \
     --mount=type=bind,src=scripts/update_assets.py,target=${HOME}/scripts/update_assets.py \
     --mount=type=bind,src=manage.py,target=${HOME}/manage.py \
 <<EOF
 make -f Makefile-docker update_assets
 EOF
 
+# This stage extends from olympia so we can run as root
+# prod_info includes production build directories to hash content
+# ensuring built assets are not modified at runtime.
+FROM olympia AS prod_info
+
+# Copy compiled locales from builder
+COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
+# Copy assets from assets
+COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-static
+
+# Build args that represent static build information
+# These are passed to docker via the bake.hcl file and
+# should not be set in the container environment.
+ARG DOCKER_COMMIT
+ARG DOCKER_VERSION
+ARG DOCKER_BUILD
+ARG DOCKER_TARGET
+
+RUN \
+    --mount=type=bind,source=scripts/build_info.py,target=${HOME}/scripts/build_info.py \
+<<EOF
+${HOME}/scripts/build_info.py \
+  --output "${BUILD_INFO}" \
+  --commit "${DOCKER_COMMIT}" \
+  --version "${DOCKER_VERSION}" \
+  --build "${DOCKER_BUILD}" \
+  --target "${DOCKER_TARGET}"
+EOF
+
 FROM base AS production
 # Copy the rest of the source files from the host
 COPY --chown=olympia:olympia . ${HOME}
@@ -193,7 +214,7 @@ COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-static
 COPY --from=assets --chown=olympia:olympia ${HOME}/static-build ${HOME}/static-build
 # Copy build info from info
-COPY --from=info ${BUILD_INFO} ${BUILD_INFO}
+COPY --from=prod_info ${BUILD_INFO} ${BUILD_INFO}
 # Copy compiled locales from builder
 COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 # Copy dependencies from `pip_production`

scripts/build_info.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+
+
+root = Path(__file__).parent.parent
+
+
+def hash_directory(
+    directory: Path,
+    exclude_patterns: list[str] = None,
+    verbose: bool = False,
+) -> str:
+    if not directory.exists():
+        return None
+
+    hasher = hashlib.blake2b(digest_size=32)
+
+    exclude_patterns = exclude_patterns or []
+
+    files = sorted(f for f in directory.rglob('*'))
+
+    for f in files:
+        is_hidden = any(part.startswith('.') for part in f.parts)
+        is_excluded = any(f.match(pattern) for pattern in exclude_patterns)
+        if not is_hidden and not is_excluded:
+            if verbose:
+                print(f'Hashing {f}')
+            hasher.update(f.name.encode())
+            if f.is_file():
+                hasher.update(f.read_bytes())
+        elif verbose:
+            print(f'Skipping {f}')
+
+    return hasher.hexdigest()
+
+
+def build_info(
+    commit: str = None, version: str = None, build: str = None, target: str = None
+):
+    """
+    Create a build info file with the current build information from the environment.
+    """
+    return {
+        'commit': commit,
+        'version': version,
+        'build': build,
+        'target': target,
+        'source': 'https://github.com/mozilla/addons-server',
+        'content_hash': {
+            'site_static_hash': hash_directory(
+                (root / 'site-static'), exclude_patterns=['staticfiles.json']
+            ),
+            'locale_hash': hash_directory((root / 'locale')),
+        },
+    }
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--output', type=str, required=False)
+
+    parser.add_argument('--commit', type=str, required=False)
+    parser.add_argument('--version', type=str, required=False)
+    parser.add_argument('--build', type=str, required=False)
+    # Docker target is used to determine build time and runtime behavior
+    parser.add_argument('--target', type=str, required=True)
+
+    args = parser.parse_args()
+
+    version = build_info(args.commit, args.version, args.build, args.target)
+
+    if args.output:
+        with open(args.output, 'w') as f:
+            json.dump(version, f)
+    else:
+        print(json.dumps(version))

src/olympia/core/apps.py

@@ -2,20 +2,18 @@
 import os
 import subprocess
 import warnings
-from io import StringIO
 from pwd import getpwnam
 
 from django.apps import AppConfig
 from django.conf import settings
 from django.core.checks import Error, Tags, register
-from django.core.management import call_command
-from django.core.management.base import CommandError
 from django.db import connection
 from django.utils.translation import gettext_lazy as _
 
 import requests
 
 from olympia.core.utils import REQUIRED_VERSION_KEYS, get_version_json
+from scripts.build_info import build_info
 
 
 log = logging.getLogger('z.startup')
@@ -85,46 +83,25 @@ def version_check(app_configs, **kwargs):
 
 @register(CustomTags.custom_setup)
 def static_check(app_configs, **kwargs):
-    errors = []
-    output = StringIO()
+    """
+    Check that the static files have not been modified since the original build.
+    """
     version = get_version_json()
 
     # We only run this check in production images.
     if version.get('target') != 'production':
         return []
 
-    try:
-        call_command('compress_assets', dry_run=True, stdout=output)
-        stripped_output = output.getvalue().strip()
-
-        if stripped_output:
-            file_paths = stripped_output.split('\n')
-            for file_path in file_paths:
-                if not os.path.exists(file_path):
-                    error = f'Compressed asset file does not exist: {file_path}'
-                    errors.append(
-                        Error(
-                            error,
-                            id='setup.E003',
-                        )
-                    )
-        else:
-            errors.append(
-                Error(
-                    'No compressed asset files were found.',
-                    id='setup.E003',
-                )
-            )
-
-    except CommandError as e:
-        errors.append(
+    site_static_hash = build_info().get('content_hash').get('site_static_hash')
+    if site_static_hash != version.get('content_hash').get('site_static_hash'):
+        return [
             Error(
-                f'Error running compress_assets command: {str(e)}',
-                id='setup.E004',
+                'Site static directory does not match expected content hash',
+                id='setup.E003',
             )
-        )
+        ]
 
-    return errors
+    return []
 
 
 @register(CustomTags.custom_setup)