Merge pull request #31 from neutrons/cron-and-logging

Add cron job to periodically purge

.github/workflows/publish.yml

@@ -83,7 +83,7 @@ jobs:
         uses: docker/build-push-action@v6
         with:
           context: .
-          file: Dockerfile
+          file: deploy/django/Dockerfile
           tags: ${{ env.REGISTRY }}/${{ github.repository }}/live_data_server:${{ steps.latest_tag.outputs.latest_tag }}
           push: true
 
@@ -92,6 +92,6 @@ jobs:
         uses: docker/build-push-action@v6
         with:
           context: .
-          file: Dockerfile
+          file: deploy/django/Dockerfile
           tags: ${{ env.REGISTRY }}/${{ github.repository }}/live_data_server:${{ steps.tag.outputs.tag }}
           push: true

.github/workflows/unittest.yml

@@ -41,7 +41,7 @@ jobs:
 
       - name: Start docker containers
         run: |
-          cp ./deploy-config/docker-compose.envlocal.yml docker-compose.yml
+          cp ./deploy/docker-compose.envlocal.yml docker-compose.yml
           docker compose up --build -d
 
       - name: Sleep, wait for containers to start up

.gitignore

@@ -64,3 +64,6 @@ target/
 
 #Ipython Notebook
 .ipynb_checkpoints
+
+# Ruff cache
+.ruff_cache

Makefile

@@ -33,7 +33,7 @@ docker/compose/validate:  ## validate the version of the docker-compose command.
 	@./scripts/docker-compose_validate.sh $(DOCKER_COMPOSE)
 
 docker/compose/local: docker/compose/validate ## compose and start the service locally
-	\cp ./deploy-config/docker-compose.envlocal.yml docker-compose.yml
+	\cp ./deploy/docker-compose.envlocal.yml docker-compose.yml
 	$(DOCKER_COMPOSE) up --build
 
 .PHONY: clean

README.md

@@ -59,7 +59,7 @@ Developer documentation at <https://livedata-ornl.readthedocs.io/en/latest/>
       make docker/compose/local
       ```
 
-      This command will copy `deploy-config/docker-compose.envlocal.yml` into `docker-compose.yml` before composing all the services.
+      This command will copy `deploy/docker-compose.envlocal.yml` into `docker-compose.yml` before composing all the services.
 
       Type `make help` to learn about other macros available as make targets.
       For instance, `make docker/pruneall` will stop all containers, then remove all containers, images, networks, and volumes.

codecov.yml

@@ -8,3 +8,5 @@ coverage:
       default:
         target: 80%
         threshold: 10%
+ignore:
+  - "src/apps/plots/migrations"

deploy/django/Dockerfile

@@ -0,0 +1,31 @@
+FROM continuumio/miniconda3:23.10.0-1 AS base
+
+### System dependencies and cron job setup
+RUN apt-get update -y && \
+    # apt upgrade -y && \
+    apt-get install -y \
+    vim cron
+
+# Set up cron job to purge expired data once a month
+COPY scripts/periodic-purge.sh /var/opt/
+RUN echo "0 0 1 * * /var/opt/periodic-purge.sh >> /var/log/cron.log 2>&1" > /etc/cron.d/root && \
+    chmod 0644 /etc/cron.d/root && \
+    crontab /etc/cron.d/root && \
+    touch /var/log/cron.log
+
+### Environment setup
+FROM base AS build
+
+COPY environment.yml .
+RUN conda env create
+
+WORKDIR /var/www/livedata
+COPY src app
+RUN mkdir ./static
+
+### Final image
+FROM build AS final
+
+COPY deploy/django/docker-entrypoint.sh /usr/bin/
+RUN chmod +x /usr/bin/docker-entrypoint.sh
+ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "livedata", "/usr/bin/docker-entrypoint.sh"]

docker-entrypoint.sh → deploy/django/docker-entrypoint.sh

@@ -1,9 +1,13 @@
 #!/bin/sh
 set -e
 
+# start cron, export root env variables
+service cron start
+env >>/etc/environment
+
 # wait for database
 until PGPASSWORD=${DATABASE_PASS} psql -h "${DATABASE_HOST}" -U "${DATABASE_USER}" -d "${DATABASE_NAME}" -c '\q'; do
-  >&2 echo "Postgres is unavailable - sleeping"
+  echo >&2 "Postgres is unavailable - sleeping"
   sleep 1
 done
 

deploy-config/docker-compose.envlocal.yml → deploy/docker-compose.envlocal.yml

@@ -7,15 +7,15 @@ services:
       - "443:443"
     volumes:
       - web-static:/var/www/livedata/static
-      - ./deploy-config/nginx/envlocal.conf:/etc/nginx/conf.d/nginx.conf
+      - ./deploy/nginx/envlocal.conf:/etc/nginx/conf.d/nginx.conf
     depends_on:
       django:
         condition: service_healthy
 
   django:
     build:
       context: .
-      dockerfile: Dockerfile
+      dockerfile: deploy/django/Dockerfile
       network: host
     environment:
       APP_DEBUG: 1  # 0 for False, otherwise will evaluate to True

docs/developer/config_for_local_use.rst

@@ -54,7 +54,7 @@ After the secrets are set, you can start the server with:
 
   make docker/compose/local
 
-This command will copy ``deploy-config/docker-compose.envlocal.yml`` into ``./docker-compose.yml`` before composing all the services.
+This command will copy ``deploy/docker-compose.envlocal.yml`` into ``./docker-compose.yml`` before composing all the services.
 
 | Run ``make help`` to learn about other macros available as make targets.
 | For instance, ``make docker/pruneall`` will stop all containers, then remove all containers, images, networks, and volumes.

docs/developer/troubleshoot/unresponsive.rst

@@ -24,9 +24,9 @@ The logs indicate a problem with the certificate files.
 An additional test is to substitute the
 `nginx.conf file for the testing environment <https://code.ornl.gov/sns-hfir-scse/deployments/livedata-deploy/-/blob/main/test/nginx.conf?ref_type=heads>`_
 with the
-`local environment one <https://github.com/neutrons/live_data_server/blob/next/deploy-config/nginx/envlocal.conf>`_,
+`local environment one <https://github.com/neutrons/live_data_server/blob/next/deploy/nginx/envlocal.conf>`_,
 which does not contain SSL certificates. Don't forget to change
-`the server name <https://github.com/neutrons/live_data_server/blob/next/deploy-config/nginx/envlocal.conf#L4>`_
+`the server name <https://github.com/neutrons/live_data_server/blob/next/deploy/nginx/envlocal.conf#L4>`_
 from `"localhost"` to `"testfixture02-test.ornl.gov"`.
 Redeploy after this. If the  http://testfixture02-test.ornl.gov/admin (notice the `http` instead of `https`) app is
 served now, then it's a problem of the secure connection.

environment.yml

@@ -11,7 +11,6 @@ dependencies:
   - psycopg=3.2
   - gunicorn
   - pytest
-  - build
   - versioningit
   - toml
   - pre-commit

scripts/periodic-purge.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+. /opt/conda/etc/profile.d/conda.sh
+printf "=%.0s" {1..88}
+printf "\nStarting periodic purge - $(date)\n"
+conda run -n livedata python /var/www/livedata/app/manage.py purge_expired_data
+printf "=%.0s" {1..88}; printf "\n"

src/apps/plots/apps.py

@@ -3,3 +3,4 @@
 
 class PlotsConfig(AppConfig):
     name = "apps.plots"
+    default_auto_field = "django.db.models.BigAutoField"

src/apps/plots/management/commands/purge_expired_data.py

@@ -8,6 +8,9 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):  # noqa: ARG002
         runs = DataRun.objects.all()
+        expired_runs = 0
         for run in runs:
             if run.expiration_date < timezone.now():
+                expired_runs += 1
                 run.delete()
+        self.stdout.write(self.style.SUCCESS(f"Deleted {expired_runs} expired runs"))

src/apps/plots/migrations/0003_alter_default_auto_fields.py

@@ -0,0 +1,27 @@
+# Generated by Django 4.2.15 on 2024-08-23 14:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("plots", "0002_datarun_expiration_date"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="datarun",
+            name="id",
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID"),
+        ),
+        migrations.AlterField(
+            model_name="instrument",
+            name="id",
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID"),
+        ),
+        migrations.AlterField(
+            model_name="plotdata",
+            name="id",
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID"),
+        ),
+    ]