Application upgrade to SQLAlchemy, Pytest

.github/workflows/pytest.yml

@@ -5,8 +5,6 @@ on:
     branches:
       - main
   pull_request:
-    branches:
-      - main
 
 jobs:
   test:
@@ -16,8 +14,20 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Start Redis
+        uses: supercharge/redis-github-action@1.7.0
+        with:
+          redis-version: 7
+
       - name: Build Docker image
         run: docker build -t glider-dac-build .
 
-      - name: Run tests
-        run: docker run --rm glider-dac-build pytest /glider-dac/tests
+      - name: Install testing requirements and run tests
+        # FIXME: Why is it necessary to set FLASK_ENV to TESTING here when
+        # pytest runs off GHA usually set this properly?
+        run: >
+          docker run --network host -e FLASK_ENV=TESTING
+          -e OVERRIDE_REDIS_HOST=localhost -e OVERRIDE_REDIS_URL=redis://localhost:6379/8
+          --rm -u root glider-dac-build bash -c
+          "pip install --no-cache -r /glider-dac/test_requirements.txt &&
+           pytest /glider-dac/glider_dac/tests"

.github/workflows/sync_theme.yml

@@ -7,7 +7,7 @@ on:
   schedule:
      - cron: "00 14 * * *"
 
-  workflow_dispatch:
+  workflow_dispatch: null
 
 
 jobs:
@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
     - name: Checkout repo
-      uses: actions/checkout@v4
+      uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4
       with: { ref: gh-pages }
 
     - name: submodule checkout
@@ -39,3 +39,13 @@ jobs:
            timestamp=$(date -u)
            git commit -m "Update theme on: ${timestamp}" || exit 0
            git push
+
+  keepalive-job:
+    name: Keepalive Workflow
+    runs-on: ubuntu-latest
+    if: github.event_name == 'schedule'
+    permissions:
+      actions: write
+    steps:
+      - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4
+      - uses: gautamkrishnar/keepalive-workflow@995aec69bb3f2b45b20f4e107907992c8715086d # 2.0.8

.gitignore

@@ -13,11 +13,10 @@ config.local.yml
 ftp_temp
 tds_temp
 tds_catalogs
-data/
+glider_dac/tests/data/
+.pytest_cache
 secrets/
 
-# BerkeleyDB user database
-users.db
 .env
 .venv/
 activate

.pre-commit-config.yaml

@@ -1,13 +1,13 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-added-large-files
 
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.6
+    rev: v2.3.0
     hooks:
       - id: codespell
         exclude: >

Dockerfile

@@ -1,31 +1,30 @@
-FROM python:3.8
+FROM python:3.9
 
 ARG glider_gid_uid=1000
 RUN apt-get update && \
     apt-get -y install cron rsync libxml2-dev libudunits2-dev \
                        libnetcdf-dev netcdf-bin && \
     mkdir glider-dac && groupadd -g $glider_gid_uid glider && \
           useradd -u $glider_gid_uid -g $glider_gid_uid glider
+ENV UDUNITS2_XML_PATH=/usr/share/xml/udunits
 COPY . /glider-dac
 # TODO: move logs elsewhere
-VOLUME /glider-dac/logs/ /data
-WORKDIR glider-dac
+VOLUME /glider-dac/logs/ /data /usr/local/lib/python3.8/site-packages/compliance_checker/data
+WORKDIR /glider-dac
 # not clear why reinstalling Mongo-related dependencies is necessary under
 # Python 3, but this allows the service to run without import or runtime errors
-RUN pip install -U pip && \
-    pip install --no-cache Cython thredds_crawler numpy==1.19.5 pytest && \
-    pip install --no-cache -r requirements.txt && \
-    pip uninstall -y mongokit && \
-    pip install --no-cache --force-reinstall mongokit-py3==0.9.1.1 && \
-    pip install -U pymongo==2.8
+RUN cd /usr/local/src && pip install -U pip && \
+    pip install --no-cache Cython thredds_crawler numpy pytest && \
+    pip install --no-cache -r /glider-dac/requirements.txt
 
 RUN mkdir -p /data/submission /data/data/priv_erddap /data/data/pub_erddap \
-             /erddapData/flag /erddapData/hardFlag berkeleydb \
+             /erddapData/flag /erddapData/hardFlag  \
              /data/catalog/priv_erddap && \
-    chown -R glider:glider /glider-dac /data && \
+    chown -R glider:glider /glider-dac /data /usr/local/lib/python3.9/site-packages/compliance_checker/data && \
     ln -sf /glider-dac/scripts/crontab /etc/crontab
 USER glider
-ENV PYTHONPATH="${PYTHONPATH}:/glider-dac"
+ENV PYTHONPATH="${PYTHONPATH:-}:/glider-dac"
+ENV FLASK_APP=glider_dac:create_app
 
 EXPOSE 5000
-CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5000", "app:app"]
+CMD ["gunicorn", "-w", "4", "-b", "0.0.0.0:5000", "glider_dac:create_app()"]

app.py

@@ -1,50 +1,49 @@
-from glider_dac import app
-from glider_dac.common import log_formatter
+from flask import current_app
 import os
 
-def initialize_logs(app):
-    '''
-    Initializes the Application Logger
-    '''
-    import logging
-    log_path = app.config.get('LOG_DIR', 'logs')
-    if not os.path.exists(log_path):
-        os.makedirs(log_path)
+#def initialize_logs(app):
+#    '''
+#    Initializes the Application Logger
+#    '''
+#    import logging
+#    log_path = app.config.get('LOG_DIR', 'logs')
+#    if not os.path.exists(log_path):
+#        os.makedirs(log_path)
+#
+#    file_handler = logging.FileHandler(os.path.join(log_path, 'application.log'))
+#    stream_handler = logging.StreamHandler()
+#    file_handler.setFormatter(log_formatter)
+#    stream_handler.setFormatter(log_formatter)
+#    app.logger.addHandler(file_handler)
+#    app.logger.addHandler(stream_handler)
+#    app.logger.setLevel(logging.DEBUG)
+#    app.logger.info('Utility Application Process Started')
 
-    file_handler = logging.FileHandler(os.path.join(log_path, 'application.log'))
-    stream_handler = logging.StreamHandler()
-    file_handler.setFormatter(log_formatter)
-    stream_handler.setFormatter(log_formatter)
-    app.logger.addHandler(file_handler)
-    app.logger.addHandler(stream_handler)
-    app.logger.setLevel(logging.DEBUG)
-    app.logger.info('Utility Application Process Started')
+#from flask import jsonify, url_for
 
-from flask import jsonify, url_for
-
-def has_no_empty_params(rule):
-    '''
-    Something to do with empty params?
-    '''
-    defaults = rule.defaults if rule.defaults is not None else ()
-    arguments = rule.arguments if rule.arguments is not None else ()
-    return len(defaults) >= len(arguments)
+#def has_no_empty_params(rule):
+#    '''
+#    Something to do with empty params?
+#    '''
+#    defaults = rule.defaults if rule.defaults is not None else ()
+#    arguments = rule.arguments if rule.arguments is not None else ()
+#    return len(defaults) >= len(arguments)
 
 #@app.route('/site-map', methods=['GET'])
-def site_map():
-    '''
-    Returns a json structure for the site routes and handlers
-    '''
-    links = []
-    for rule in app.url_map.iter_rules():
-        # Filter out rules we can't navigate to in a browser
-        # and rules that require parameters
-        if "GET" in rule.methods and has_no_empty_params(rule):
-            url = url_for(rule.endpoint)
-            links.append((url, rule.endpoint))
-    # links is now a list of url, endpoint tuples
-    return jsonify(rules=links)
-initialize_logs(app)
+#def site_map():
+#    '''
+#    Returns a json structure for the site routes and handlers
+#    '''
+#    links = []
+#    for rule in app.url_map.iter_rules():
+#        # Filter out rules we can't navigate to in a browser
+#        # and rules that require parameters
+#        if "GET" in rule.methods and has_no_empty_params(rule):
+#            url = url_for(rule.endpoint)
+#            links.append((url, rule.endpoint))
+#    # links is now a list of url, endpoint tuples
+#    return jsonify(rules=links)
+#initialize_logs(app)
 
 if __name__ == '__main__':
     app.run(host=app.config['HOST'], port=app.config['PORT'], debug=app.config['DEBUG'])

config.yml

@@ -1,12 +1,12 @@
 COMMON: &common
+  FLASK_APP: "glider_dac:create_app()"
   HOST: localhost
   PORT: 5000
   DEBUG: False
-  MONGODB_HOST: mongo
-  MONGODB_PORT: 27017
-  MONGODB_DATABASE: gliderdac
 
-  APPLICATION_PREFIX: "/gliders/"
+  SERVER_NAME: "localhost:5000"
+  APPLICATION_ROOT: "/"
+  APPLICATION_PREFIX: "/"
   SECRET_KEY: thisisakey
   LOG_FILE: yes
   WEB_PASSWORD: password
@@ -25,6 +25,11 @@ COMMON: &common
   MAILER_DEBUG: 1
   MAIL_DEFAULT_TO: receiver@domain.com
   #MAIL_DEFAULT_LIST: YOUR_EMAIL
+  #
+
+  # IMPORTANT: Modification tracking must be set to True for certain parts of
+  # deployment lifecycle to work properly!
+  SQLALCHEMY_TRACK_MODIFICATIONS: True
 
   # Google Analytics
   GA_ENABLED: True
@@ -43,7 +48,6 @@ COMMON: &common
   INFLUXDB_PORT: 8086
   ADMINS:
    - admin
-  USER_DB_FILE: berkeleydb/users.db
 
   REDIS_HOST: redis
   REDIS_PORT: 6379
@@ -69,6 +73,34 @@ COMMON: &common
   PUBLIC_CATALOG: '/data/catalog/pub_erddap/datasets.xml'
   PRIVATE_CATALOG: '/data/catalog/priv_erddap/datasets.xml'
 
+  SQLALCHEMY_DATABASE_URI: mysql://username:password@mysql/gliderdac
+
 DEVELOPMENT: &development
   <<: *common
   DEBUG: True
+  PREFERRED_URL_SCHEME: http
+
+TESTING:
+  <<: *common
+  DEBUG: True
+  TESTING: True
+  PREFERRED_URL_SCHEME: http
+  # enable email for mock email testing
+  MAIL_ENABLED: True
+  # only use in memory DB for testing
+  SQLALCHEMY_DATABASE_URI: 'sqlite:///:memory:'
+  WTF_CSRF_ENABLED: False
+  # TODO: these directories need to be made relative to project root
+  path2priv: 'tests/test_fs/data/priv_erddap/'
+  path2pub: 'tests/test_fs/data/pub_erddap/'
+  path2thredds: 'tests/test_fs/data/data/thredds/'
+  flags_private: 'tests/test_fs/scratch/tomcat-erddap-private/flag'
+  flags_public: 'tests/test_fs/scratch/tomcat-erddap-public/flag'
+  DATA_ROOT: tests/test_fs/submission
+  PRIV_DATA_ROOT: tests/test_fs/data/priv_erddap
+  PUBLIC_DATA_ROOT: tests/test_fs/data/pub_erddap
+  THREDDS_DATA_ROOT: tests/test_fs/data/thredds
+  ARCHIVE_PATH: tests/test_fs/data/archive
+  NCEI_DIR: tests/test_fs/data/archive
+  SERVER: "http://localhost:8080/erddap"
+  erddap_private: 'localhost:8080'

console

@@ -1,8 +1,9 @@
 #!/usr/bin/env python
 
 from IPython import embed
-from glider_dac import app, db
-from bson import ObjectId
+from glider_dac import db
+from flask import current_app
+from glider_dac.extensions import db
 
-with app.app_context():
+with current_app.app_context():
     embed()