From bb897246cb830cafa31982f7cc7418979bd249e4 Mon Sep 17 00:00:00 2001 From: joaquin Date: Thu, 8 Feb 2024 13:28:05 -0500 Subject: [PATCH] [MRG] Optimize Dockerfiles (#241) * Remove unnecessary APPLE_SILICON variable from Makefiles. * Optimize Dockefiles --------- Co-authored-by: mgonnav --- development-helps/Makefile | 14 +++--- .../docker-conf/Dockerfile-build-project | 48 ++++++++---------- estela-api/docker-conf/Dockerfile-celery-beat | 49 +++++++++++++------ .../docker-conf/Dockerfile-celery-worker | 47 +++++++++++++----- estela-api/docker-conf/Dockerfile-django-api | 47 +++++++++++++----- installation/Makefile | 5 +- queueing/Dockerfile | 46 ++++++++++++----- 7 files changed, 167 insertions(+), 89 deletions(-) diff --git a/development-helps/Makefile b/development-helps/Makefile index 8b470393..f9252d95 100644 --- a/development-helps/Makefile +++ b/development-helps/Makefile @@ -8,7 +8,7 @@ REGISTRY_HOST = localhost:5001 API_POD = $$(kubectl get pod -l app=estela-django-api -o jsonpath="{.items[0].metadata.name}") API_DOC = docs/api.yaml PLATFORM ?= linux/$(shell uname -m) -APPLE_SILICON ?= $$(uname -m | grep -q 'arm64' && echo "true" || echo "false") + .PHONY: start start: @@ -23,15 +23,15 @@ stop: .PHONY: update-api-image update-api-image: -cd $(API_DIR) && \ - docker build .. --build-arg APPLE_SILICON=${APPLE_SILICON} --file docker-conf/Dockerfile-django-api --tag $(REGISTRY_HOST)/estela-django-api:latest --platform $(PLATFORM) + docker build .. --file docker-conf/Dockerfile-django-api --tag $(REGISTRY_HOST)/estela-django-api:latest --platform $(PLATFORM) -docker push $(REGISTRY_HOST)/estela-django-api:latest .PHONY: update-celery-image update-celery-image: -cd $(API_DIR) && \ - docker build .. --build-arg APPLE_SILICON=${APPLE_SILICON} --file docker-conf/Dockerfile-celery-beat --tag $(REGISTRY_HOST)/estela-celery-beat:latest --platform $(PLATFORM) && \ - docker build .. --build-arg APPLE_SILICON=${APPLE_SILICON} --file docker-conf/Dockerfile-celery-worker --tag $(REGISTRY_HOST)/estela-celery-worker:latest --platform $(PLATFORM) + docker build .. --file docker-conf/Dockerfile-celery-beat --tag $(REGISTRY_HOST)/estela-celery-beat:latest --platform $(PLATFORM) && \ + docker build .. --file docker-conf/Dockerfile-celery-worker --tag $(REGISTRY_HOST)/estela-celery-worker:latest --platform $(PLATFORM) -docker push $(REGISTRY_HOST)/estela-celery-beat:latest -docker push $(REGISTRY_HOST)/estela-celery-worker:latest @@ -39,21 +39,21 @@ update-celery-image: .PHONY: update-redis-image update-redis-image: -cd $(API_DIR) && \ - docker build . --build-arg APPLE_SILICON=${APPLE_SILICON} --file docker-conf/Dockerfile-redis --tag $(REGISTRY_HOST)/estela-redis:latest --platform $(PLATFORM) + docker build . --file docker-conf/Dockerfile-redis --tag $(REGISTRY_HOST)/estela-redis:latest --platform $(PLATFORM) -docker push $(REGISTRY_HOST)/estela-redis:latest .PHONY: update-build-project-image update-build-project-image: -cd $(API_DIR) && \ - docker build .. --build-arg APPLE_SILICON=${APPLE_SILICON} --file docker-conf/Dockerfile-build-project --tag $(REGISTRY_HOST)/estela-build-project:latest --platform $(PLATFORM) + docker build .. --file docker-conf/Dockerfile-build-project --tag $(REGISTRY_HOST)/estela-build-project:latest --platform $(PLATFORM) -docker push $(REGISTRY_HOST)/estela-build-project:latest .PHONY: update-consumer-image update-consumer-image: -cd $(QUEUING_DIR) && \ - docker build .. --build-arg APPLE_SILICON=${APPLE_SILICON} --file Dockerfile --tag $(REGISTRY_HOST)/estela-consumer:latest --platform $(PLATFORM) + docker build .. --file Dockerfile --tag $(REGISTRY_HOST)/estela-consumer:latest --platform $(PLATFORM) -docker push $(REGISTRY_HOST)/estela-consumer:latest diff --git a/estela-api/docker-conf/Dockerfile-build-project b/estela-api/docker-conf/Dockerfile-build-project index ec8e333b..cd4e3174 100644 --- a/estela-api/docker-conf/Dockerfile-build-project +++ b/estela-api/docker-conf/Dockerfile-build-project @@ -1,40 +1,32 @@ -FROM python:3.9 +FROM python:3.9-slim WORKDIR /home/estela -ARG APPLE_SILICON=false -COPY estela-api/requirements ./requirements - -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONUNBUFFERED 1 +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 -RUN apt-get update - -RUN apt-get install \ +RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ + build-essential \ curl \ + git \ gnupg \ - lsb-release -y - -RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + lsb-release \ + unixodbc-dev \ + default-libmysqlclient-dev \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(lsb_release -cs) stable" > /etc/apt/sources.list.d/docker.list \ + && apt-get update \ + && apt-get install -y docker-ce docker-ce-cli containerd.io \ + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* -RUN echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ - focal stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null - -RUN apt-get update - -RUN apt-get install docker-ce docker-ce-cli containerd.io -y - -# If an Apple Silicon chip is used, install pyodbc and unixodbc-dev -RUN if [ "$APPLE_SILICON" = "true" ]; then \ - apt-get update && apt-get install -y unixodbc-dev && \ - pip install --no-binary :all: pyodbc && \ - sed -i '/pyodbc/d' requirements/deploy.txt; \ - fi +COPY estela-api/requirements ./requirements -RUN pip install -r requirements/deploy.txt -RUN if test -f "requirements/externalApps.txt"; then pip install -r requirements/externalApps.txt; fi +# Install Python dependencies +RUN pip install --no-cache-dir -r requirements/deploy.txt \ + && { [ -f requirements/externalApps.txt ] && pip install --no-cache-dir -r requirements/externalApps.txt || true; } COPY estela-api/ estela-api/ COPY database_adapters/ estela-api/database_adapters/ diff --git a/estela-api/docker-conf/Dockerfile-celery-beat b/estela-api/docker-conf/Dockerfile-celery-beat index 35e41755..5a09c287 100644 --- a/estela-api/docker-conf/Dockerfile-celery-beat +++ b/estela-api/docker-conf/Dockerfile-celery-beat @@ -1,23 +1,44 @@ -FROM python:3.9 +# Build Stage +FROM python:3.9-slim as builder -ARG APPLE_SILICON=false +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 -WORKDIR /home/estela +WORKDIR /build + +# Install build dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + build-essential \ + unixodbc-dev \ + default-libmysqlclient-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* COPY estela-api/requirements ./requirements -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONUNBUFFERED 1 +RUN pip install --no-cache-dir --target=/build/deps -r requirements/test.txt \ + && if [ -f requirements/externalApps.txt ]; then pip install --no-cache-dir --target=/build/deps -r requirements/externalApps.txt; fi + + +# Final Stage +FROM python:3.9-slim -# If an Apple Silicon chip is used, install pyodbc and unixodbc-dev -RUN if [ "$APPLE_SILICON" = "true" ]; then \ - apt-get update && apt-get install -y unixodbc-dev && \ - pip install --no-binary :all: pyodbc && \ - sed -i '/pyodbc/d' requirements/test.txt; \ - fi - -RUN pip install -r requirements/test.txt -RUN if test -f "requirements/externalApps.txt"; then pip install -r requirements/externalApps.txt; fi +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /home/estela + +# Copy installed dependencies from the builder stage +COPY --from=builder /build/deps /usr/local/lib/python3.9/site-packages +COPY --from=builder /build/deps/bin /usr/local/bin COPY estela-api/ . COPY database_adapters/ ./database_adapters + +# Runtime dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + unixodbc-dev \ + default-libmysqlclient-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* diff --git a/estela-api/docker-conf/Dockerfile-celery-worker b/estela-api/docker-conf/Dockerfile-celery-worker index 4f469fb9..5a09c287 100644 --- a/estela-api/docker-conf/Dockerfile-celery-worker +++ b/estela-api/docker-conf/Dockerfile-celery-worker @@ -1,23 +1,44 @@ -FROM python:3.9 +# Build Stage +FROM python:3.9-slim as builder -ARG APPLE_SILICON=false +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 -WORKDIR /home/estela +WORKDIR /build + +# Install build dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + build-essential \ + unixodbc-dev \ + default-libmysqlclient-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* COPY estela-api/requirements ./requirements -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONUNBUFFERED 1 +RUN pip install --no-cache-dir --target=/build/deps -r requirements/test.txt \ + && if [ -f requirements/externalApps.txt ]; then pip install --no-cache-dir --target=/build/deps -r requirements/externalApps.txt; fi -# If an Apple Silicon chip is used, install pyodbc and unixodbc-dev -RUN if [ "$APPLE_SILICON" = "true" ]; then \ - apt-get update && apt-get install -y unixodbc-dev && \ - pip install --no-binary :all: pyodbc && \ - sed -i '/pyodbc/d' requirements/test.txt; \ - fi -RUN pip install -r requirements/test.txt -RUN if test -f "requirements/externalApps.txt"; then pip install -r requirements/externalApps.txt; fi +# Final Stage +FROM python:3.9-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /home/estela + +# Copy installed dependencies from the builder stage +COPY --from=builder /build/deps /usr/local/lib/python3.9/site-packages +COPY --from=builder /build/deps/bin /usr/local/bin COPY estela-api/ . COPY database_adapters/ ./database_adapters + +# Runtime dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + unixodbc-dev \ + default-libmysqlclient-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* diff --git a/estela-api/docker-conf/Dockerfile-django-api b/estela-api/docker-conf/Dockerfile-django-api index 33dbb9ba..c87c7933 100644 --- a/estela-api/docker-conf/Dockerfile-django-api +++ b/estela-api/docker-conf/Dockerfile-django-api @@ -1,24 +1,45 @@ -FROM python:3.9 +# Build Stage +FROM python:3.9-slim as builder -WORKDIR /home/estela -ARG APPLE_SILICON=false +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /build + +# Install build dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + build-essential \ + unixodbc-dev \ + default-libmysqlclient-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* COPY estela-api/requirements ./requirements -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONUNBUFFERED 1 +RUN pip install --no-cache-dir --target=/build/deps -r requirements/test.txt \ + && if [ -f requirements/externalApps.txt ]; then pip install --no-cache-dir --target=/build/deps -r requirements/externalApps.txt; fi + -# If an Apple Silicon chip is used, install pyodbc and unixodbc-dev -RUN if [ "$APPLE_SILICON" = "true" ]; then \ - apt-get update && apt-get install -y unixodbc-dev && \ - pip install --no-binary :all: pyodbc && \ - sed -i '/pyodbc/d' requirements/test.txt; \ - fi +# Final Stage +FROM python:3.9-slim -RUN pip install -r requirements/test.txt -RUN if test -f "requirements/externalApps.txt"; then pip install -r requirements/externalApps.txt; fi +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /home/estela + +# Copy installed dependencies from the builder stage +COPY --from=builder /build/deps /usr/local/lib/python3.9/site-packages COPY estela-api/ . COPY database_adapters/ ./database_adapters +# Runtime dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + unixodbc-dev \ + default-libmysqlclient-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + EXPOSE 8000 diff --git a/installation/Makefile b/installation/Makefile index 2b6b6813..8f43fe3c 100644 --- a/installation/Makefile +++ b/installation/Makefile @@ -7,7 +7,6 @@ LOCAL_API_IP = $$(kubectl get services -n $${NAMESPACE} estela-django-api-servic RESOURCES = db registry minio zookeeper kafka SERVICES ?= django-api celery-worker celery-beat redis build-project PLATFORM ?= linux/$(shell uname -m) -APPLE_SILICON ?= $$(uname -m | grep -q 'arm64' && echo "true" || echo "false") .PHONY: resources @@ -34,9 +33,9 @@ delete-resources: .PHONY: build-all-images build-all-images: -. ./local/.env && for service in $(SERVICES); do \ - cd $(API_DIR) && docker build .. --build-arg APPLE_SILICON=${APPLE_SILICON} --file docker-conf/Dockerfile-$$service --tag $${LOCAL_REGISTRY}/estela-$$service:latest; \ + cd $(API_DIR) && docker build .. --file docker-conf/Dockerfile-$$service --tag $${LOCAL_REGISTRY}/estela-$$service:latest; \ done - -. ./local/.env && cd $(QUEUING_DIR) && docker build .. --build-arg APPLE_SILICON=${APPLE_SILICON} --file Dockerfile --tag $${LOCAL_REGISTRY}/estela-consumer:latest + -. ./local/.env && cd $(QUEUING_DIR) && docker build .. --file Dockerfile --tag $${LOCAL_REGISTRY}/estela-consumer:latest .PHONY: upload-all-images upload-all-images: diff --git a/queueing/Dockerfile b/queueing/Dockerfile index 72142360..354b77fe 100644 --- a/queueing/Dockerfile +++ b/queueing/Dockerfile @@ -1,21 +1,45 @@ -FROM python:3.9 +# Build Stage +FROM python:3.9-slim as builder -ARG APPLE_SILICON=false -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONUNBUFFERED 1 +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 -WORKDIR /home/estela +WORKDIR /build + +# Install build dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + build-essential \ + unixodbc-dev \ + default-libmysqlclient-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* COPY queueing/requirements requirements -RUN if [ "$APPLE_SILICON" = "true" ]; then \ - apt-get update && apt-get install -y unixodbc-dev && \ - pip install --no-binary :all: pyodbc && \ - sed -i '/pyodbc/d' requirements/consumer.txt; \ - fi -RUN pip install -r requirements/consumer.txt + +RUN pip install --no-cache-dir --target=/build/deps -r requirements/consumer.txt + + +# Final Stage +FROM python:3.9-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /home/estela + +# Copy installed dependencies from the builder stage +COPY --from=builder /build/deps /usr/local/lib/python3.9/site-packages COPY queueing/consumer.py . COPY queueing/inserter.py . COPY queueing/utils.py . COPY queueing/config config COPY database_adapters/ ./database_adapters + +# Runtime dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + unixodbc-dev \ + default-libmysqlclient-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/*