Switch to pipenv (#449)

* remove django countries

* convert requirements to pipenv

* update dockerfile

* add productions docker tags

* update direnv config

.envrc

@@ -1,2 +1,2 @@
-source .venv/bin/activate; unset PS1
+layout pipenv
 dotenv

.github/workflows/deployment.yaml

@@ -5,12 +5,12 @@ on:
   push:
     branches:
       - master
-
+      - production
 
 jobs:
   build:
     name: Build & Push Staging to container registries
-    if: github.ref == 'refs/heads/master'
+    if: ${{ github.ref == 'refs/heads/master' || github.ref == 'refs/heads/production' }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -26,16 +26,19 @@ jobs:
             type=raw,value=latest-${{ github.run_number }}
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
+            type=raw,value=production-latest,enable=${{ github.ref == 'refs/heads/production' }}
+            type=raw,value=production-latest-${{ github.run_number }}-{{date 'YYYYMMDD'}}-{{sha}},enable=${{ github.ref == 'refs/heads/production' }}
           flavor: |
             latest=true
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
       - name: Cache Docker layers
         uses: actions/cache@v3
         with:
           path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ hashFiles('r*/base.txt', 'r*/production.txt', 'Dockerfile') }}
+          key: ${{ runner.os }}-buildx-${{ hashFiles('Pipenv.lock', 'compose/production/django/Dockerfile') }}
           restore-keys: |
             ${{ runner.os }}-buildx-
 
@@ -44,7 +47,7 @@ jobs:
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
-          
+
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v3
         with:

Makefile

@@ -30,11 +30,11 @@ logs:
 	docker compose -f $(docker_config_file) logs
 
 makemigrations: up
-	docker exec care bash -c "python manage.py makemigrations"
+	docker exec django bash -c "python manage.py makemigrations"
 
 test: up
-	docker exec care bash -c "python manage.py test --keepdb --parallel=$(nproc)"
+	docker exec django bash -c "python manage.py test --keepdb --parallel=$(nproc)"
 
 test_coverage: up
-	docker exec care bash -c "coverage run manage.py test --settings=config.settings.test --keepdb --parallel=$(nproc)"
-	docker exec care bash -c "coverage combine || true; coverage report"
+	docker exec django bash -c "coverage run manage.py test --settings=config.settings.test --keepdb --parallel=$(nproc)"
+	docker exec django bash -c "coverage combine || true; coverage report"

Pipfile

@@ -0,0 +1,57 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+pillow = "==10.0.0"
+argon2-cffi = "==23.1.0"
+whitenoise = "==6.6.0"
+redis = "==5.0.1"
+celery = "==5.3.6"
+django = "==4.2.6"
+djangoql = "==0.17.1"
+djangorestframework = "==3.14.0"
+django-anymail = {extras = ["mailgun"], version = "==10.2"}
+django-cors-headers = "==4.2.0"
+django-environ = "==0.11.2"
+django-filter = "==23.2"
+django-model-utils = "==4.3.1"
+django-simple-history = "==3.4.0"
+django-storages = {extras = ["s3"], version = "==1.14.2"}
+drf-nested-routers = "==0.93.4"
+drf-spectacular = "==0.27.0"
+requests = "==2.31.0"
+openai = "==1.2.4"
+pinecone-client = "==2.2.4"
+pypdf2 = "==3.0.1"
+tiktoken = "==0.5.2"
+langchain = "==0.0.352"
+langchain-experimental = "==0.0.40"
+anyio = "<4.0.0" # Do not update: anyio<4.0 is REQUIRED by langchain
+google-api-core = "==2.11.1"
+google-auth = "==2.23.4"
+google-cloud-core = "==2.3.3"
+google-cloud-speech = "==2.21.0"
+google-cloud-texttospeech = "==2.14.2"
+google-cloud-translate = "==3.12.0"
+googleapis-common-protos = "==1.60.0"
+nltk = "==3.8.1"
+gunicorn = "==21.2.0"
+psycopg = {extras = ["c"], version = "==3.1.17"}
+sentry-sdk = "==1.30.0"
+
+[dev-packages]
+ipdb = "==0.13.13"
+flake8 = "==6.1.0"
+flake8-isort = "==6.0.0"
+coverage = "==7.4.1"
+black = "==24.1.1"
+pre-commit = "==3.6.0"
+factory-boy = "==3.3.0"
+django-debug-toolbar = "==4.2.0"
+django-extensions = "==3.2.3"
+psycopg = {extras = ["binary"], version = "==3.1.17"}
+
+[requires]
+python_version = "3.11"

compose/local/django/Dockerfile

@@ -1,4 +1,4 @@
-ARG PYTHON_VERSION=3.9-slim-bullseye
+ARG PYTHON_VERSION=3.11-slim-bullseye
 
 # define an alias for the specfic python version used in this file.
 FROM python:${PYTHON_VERSION} as python
@@ -15,13 +15,12 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
   # psycopg2 dependencies
   libpq-dev
 
-# Requirements are installed here to ensure they will be cached.
-COPY ./requirements .
-
-# Create Python Dependency and Sub-Dependency Wheels.
-RUN pip wheel --wheel-dir /usr/src/app/wheels  \
-  -r ${BUILD_ENVIRONMENT}.txt
+RUN python -m venv /venv
+RUN pip install pipenv
 
+# Requirements are installed here to ensure they will be cached.
+COPY Pipfile Pipfile.lock ./
+RUN pipenv sync --system --categories "packages"
 
 # Python 'run' stage
 FROM python as python-run-stage
@@ -32,6 +31,7 @@ ARG APP_HOME=/app
 ENV PYTHONUNBUFFERED 1
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV BUILD_ENV ${BUILD_ENVIRONMENT}
+ENV PATH /venv/bin:$PATH
 
 WORKDIR ${APP_HOME}
 
@@ -45,13 +45,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
   && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
   && rm -rf /var/lib/apt/lists/*
 
-# All absolute dir copies ignore workdir instruction. All relative dir copies are wrt to the workdir instruction
-# copy python dependency wheels from python-build-stage
-COPY --from=python-build-stage /usr/src/app/wheels  /wheels/
-
-# use wheels to install python dependencies
-RUN pip install --no-cache-dir --no-index --find-links=/wheels/ /wheels/* \
-	&& rm -rf /wheels/
+COPY --from=python-build-stage /venv /venv
 
 COPY ./compose/production/django/entrypoint /entrypoint
 RUN sed -i 's/\r$//g' /entrypoint
@@ -61,7 +55,6 @@ COPY ./compose/local/django/start /start
 RUN sed -i 's/\r$//g' /start
 RUN chmod +x /start
 
-
 COPY ./compose/local/django/celery/worker/start /start-celeryworker
 RUN sed -i 's/\r$//g' /start-celeryworker
 RUN chmod +x /start-celeryworker

compose/production/django/Dockerfile

@@ -1,6 +1,4 @@
-ARG PYTHON_VERSION=3.9-slim-bullseye
-
-
+ARG PYTHON_VERSION=3.11-slim-bullseye
 
 # define an alias for the specfic python version used in this file.
 FROM python:${PYTHON_VERSION} as python
@@ -17,13 +15,12 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
   # psycopg2 dependencies
   libpq-dev
 
-# Requirements are installed here to ensure they will be cached.
-COPY ./requirements .
-
-# Create Python Dependency and Sub-Dependency Wheels.
-RUN pip wheel --wheel-dir /usr/src/app/wheels  \
-  -r ${BUILD_ENVIRONMENT}.txt
+RUN python -m venv /venv
+RUN pip install pipenv
 
+# Requirements are installed here to ensure they will be cached.
+COPY Pipfile Pipfile.lock ./
+RUN pipenv sync --system --categories "packages"
 
 # Python 'run' stage
 FROM python as python-run-stage
@@ -34,56 +31,39 @@ ARG APP_HOME=/app
 ENV PYTHONUNBUFFERED 1
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV BUILD_ENV ${BUILD_ENVIRONMENT}
+ENV PATH /venv/bin:$PATH
 
 WORKDIR ${APP_HOME}
 
-RUN addgroup --system django \
-  && adduser --system --ingroup django django
-
-
 # Install required system dependencies
 RUN apt-get update && apt-get install --no-install-recommends -y \
-  # psycopg2 dependencies
+  # psycopg dependencies
   libpq-dev \
   # Translations dependencies
   gettext \
   # cleaning up unused files
   && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
   && rm -rf /var/lib/apt/lists/*
 
-# All absolute dir copies ignore workdir instruction. All relative dir copies are wrt to the workdir instruction
-# copy python dependency wheels from python-build-stage
-COPY --from=python-build-stage /usr/src/app/wheels  /wheels/
+COPY --from=python-build-stage /venv /venv
 
-# use wheels to install python dependencies
-RUN pip install --no-cache-dir --no-index --find-links=/wheels/ /wheels/* \
-  && rm -rf /wheels/
-
-
-COPY --chown=django:django ./compose/production/django/entrypoint /entrypoint
+COPY ./compose/production/django/entrypoint /entrypoint
 RUN sed -i 's/\r$//g' /entrypoint
 RUN chmod +x /entrypoint
 
-
-COPY --chown=django:django ./compose/production/django/start /start
+COPY ./compose/production/django/start /start
 RUN sed -i 's/\r$//g' /start
 RUN chmod +x /start
-COPY --chown=django:django ./compose/production/django/celery/worker/start /start-celeryworker
+
+COPY ./compose/production/django/celery/worker/start /start-celeryworker
 RUN sed -i 's/\r$//g' /start-celeryworker
 RUN chmod +x /start-celeryworker
 
-
-COPY --chown=django:django ./compose/production/django/celery/beat/start /start-celerybeat
+COPY ./compose/production/django/celery/beat/start /start-celerybeat
 RUN sed -i 's/\r$//g' /start-celerybeat
 RUN chmod +x /start-celerybeat
 
-
 # copy application code to WORKDIR
-# COPY --chown=django:django . ${APP_HOME}
 COPY  . ${APP_HOME}
-# # make django owner of the WORKDIR directory as well.
-# RUN chown django:django ${APP_HOME}
-
-# USER django
 
 ENTRYPOINT ["/entrypoint"]

core/settings/base.py

@@ -79,7 +79,6 @@
     "django_filters",
     "drf_spectacular",
     "djangoql",
-    "django_countries",
     "simple_history",
 ]
 

docker-compose.local.yaml

@@ -11,7 +11,6 @@ services:
       dockerfile: ./compose/local/django/Dockerfile
     image: django
     container_name: ayushma_local_django
-    platform: linux/x86_64
     depends_on:
       - postgres
       - redis

docker-compose.production.yaml

@@ -10,13 +10,14 @@ services:
       context: .
       dockerfile: ./compose/production/django/Dockerfile
     image: ayushma_production_django
-    platform: linux/x86_64
     depends_on:
       - postgres
       - redis
     env_file:
       - ./.envs/.production/.django
       - ./.envs/.production/.postgres
+    ports:
+      - "8000:8000"
     command: /start
 
   postgres:
@@ -36,9 +37,11 @@ services:
   celeryworker:
     <<: *django
     image: ayushma_production_celeryworker
+    ports: []
     command: /start-celeryworker
 
   celerybeat:
     <<: *django
     image: ayushma_production_celerybeat
+    ports: []
     command: /start-celerybeat