Elasticsearch 8, Python 3.8.10, pygeoapi and Docker updates

.github/workflows/flake8.yml

@@ -7,11 +7,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.8]
 
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
       name: Set up Python ${{ matrix.python-version }}
       with:
         python-version: ${{ matrix.python-version }}
@@ -20,6 +20,6 @@ jobs:
         python3 -m pip install --upgrade pip
         pip3 install flake8
     - name: run flake8 ⚙️
-      run: | 
-        flake8
+      run: |
+        flake8 .
 

Dockerfile

@@ -0,0 +1,84 @@
+###################################################################
+#
+# Author: Kevin Ngai <kevin.ngai@ec.gc.ca>
+#
+# Copyright (c) 2024 Kevin Ngai
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+###################################################################
+
+# Development stage
+FROM ubuntu:jammy AS develop-stage
+
+# Default environment variables if none are passed in
+ARG PYGEOAPI_GITREPO=https://github.com/geopython/pygeoapi.git
+ARG WOUDC_EXTCSV_GITREPO=https://github.com/woudc/woudc-extcsv.git
+
+# Set environment variables
+ENV BASEDIR=/data/web/woudc-api-nightly
+ENV APPDIR=${BASEDIR}/woudc-api
+ENV DEBIAN_FRONTEND=noninteractive
+
+WORKDIR ${BASEDIR}
+
+# Set up the repository mirror for faster package retrieval
+RUN sed -i 's/http:\/\/archive.ubuntu.com\/ubuntu\//mirror:\/\/mirrors.ubuntu.com\/mirrors.txt/g' /etc/apt/sources.list && \
+    apt-get update && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository ppa:gcpp-kalxas/wmo && \
+    add-apt-repository ppa:ubuntugis/ppa && \
+    apt-get update && \
+    apt-get install -y python3 python3-pip python3-flask git curl unzip && \
+    # Install pygeoapi
+    git clone ${PYGEOAPI_GITREPO} -b master --depth=1 && \
+    cd pygeoapi && \
+    pip3 install -r requirements.txt && \
+    pip3 install . && \
+    cd ${BASEDIR} && \
+    # Install woudc-extcsv
+    git clone ${WOUDC_EXTCSV_GITREPO} -b master --depth=1 && \
+    cd woudc-extcsv && \
+    pip3 install . && \
+    cd ${BASEDIR}
+
+# Copy application code
+COPY . ${APPDIR}
+
+# Install application dependencies
+RUN cd ${APPDIR} && \
+    pip3 install -r requirements.txt && \
+    pip3 install gunicorn gevent Flask-Cors && \
+    pip3 install .
+
+# Cleanup unnecessary packages and files
+RUN apt-get remove --purge -y git && \
+    apt-get clean && \
+    apt autoremove -y && \
+    rm -rf /var/lib/apt/lists/*
+
+# Fix permissions (mark entrypoint as executable)
+RUN chmod +x ${APPDIR}/entrypoint.sh
+
+# Start entrypoint.sh
+ENTRYPOINT /bin/sh -c "${APPDIR}/entrypoint.sh"
+

README.md

@@ -27,32 +27,42 @@ are automatically installed during installation.
 
 ```bash
 # setup virtualenv
-python3 -m venv --system-site-packages woudc-api
-cd woudc-api
+python3 -m venv --system-site-packages woudc-api_env
+cd woudc-api_env
 source bin/activate
 
 # setup local OGC schemas (i.e. WOUDC_API_OGC_SCHEMAS_LOCATION in default.env)
 mkdir schemas.opengis.net
 curl -O http://schemas.opengis.net/SCHEMAS_OPENGIS_NET.zip && unzip ./SCHEMAS_OPENGIS_NET.zip "ogcapi/*" -d schemas.opengis.net && rm -f ./SCHEMAS_OPENGIS_NET.zip
 
-# clone pygeoapi codebase and install
+# optional for development: clone pygeoapi codebase and install
 git clone https://github.com/geopython/pygeoapi.git
 cd pygeoapi
-python setup.py install
+pip3 install -r requirements.txt
+python3 setup.py install
+cd ..
+
+# clone woudc-extcsv and install
+git clone https://github.com/woudc/woudc-extcsv.git
+cd woudc-extcsv
+python3 setup.py install
 cd ..
 
 # clone woudc-api codebase and install
 git clone https://github.com/woudc/woudc-api.git
 cd woudc-api
-python setup.py install
+python3 setup.py install
 
 # set system environment variables
 cp default.env local.env
 vi local.env  # edit accordingly
 . local.env
 
 # generate openapi document
-pygeoapi openapi generate -c $PYGEOAPI_CONFIG > $PYGEOAPI_OPENAPI
+pygeoapi openapi generate ${PYGEOAPI_CONFIG} -f json --output-file ${PYGEOAPI_OPENAPI}
+
+# optional: validate openapi document
+pygeoapi openapi validate ${PYGEOAPI_OPENAPI}
 
 # run the server
 woudc-api serve  # server runs on http://localhost:5000
@@ -62,6 +72,39 @@ curl http://localhost:5000  # redirect to WOUDC data services pages
 curl http://localhost:5000/oapi  # OGC API endpoint
 ```
 
+#### Docker
+
+Docker commands:
+```bash
+# set system environment variables
+cp default.env local.env
+vi local.env  # edit accordingly
+. local.env
+
+# build
+docker build -t woudc-api .
+
+# run container
+docker run -d --name woudc-api -p ${WOUDC_API_BIND_PORT}:${WOUDC_API_BIND_PORT} woudc-api
+```
+
+Docker compose commands (recommended):
+```bash
+# set system environment variables
+cp default.env local.env
+vi local.env  # edit accordingly
+. local.env
+
+# take down container
+docker compose -f docker-compose.yml down
+
+# build
+docker compose -f docker-compose.yml build
+
+# run container (detached)
+docker compose -f docker-compose.yml up -d
+```
+
 ### Development
 
 ```bash

debian/control

@@ -4,12 +4,12 @@ Priority: optional
 Maintainer: Tom Kralidis <tom.kralidis@canada.ca>
 Build-Depends: debhelper (>= 9), python3, python3-setuptools, curl, unzip
 Standards-Version: 3.9.5
-X-Python-Version: >= 3.6
+X-Python-Version: >= 3.8
 Vcs-Git: https://github.com/woudc/woudc-api.git
 
 Package: woudc-api
 Architecture: all
-Depends: elasticsearch (>=7), elasticsearch (<8), libapache2-mod-wsgi-py3, python3, python3-flask, python3-pygeoapi, woudc-extcsv
-Suggests: python3-elasticsearch (>=7), python3-elasticsearch (<8)
+Depends: elasticsearch (>=8), elasticsearch (<9), libapache2-mod-wsgi-py3, python3, python3-flask, python3-pygeoapi, woudc-extcsv
+Suggests: python3-elasticsearch (>=8), python3-elasticsearch (<9)
 Homepage: https://woudc.org
 Description: WMO WOUDC REST API

debian/postinst

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-set -e 
+set -e
 
 USERNAME="woudcuser"
 GROUP="www-data"

default.env

@@ -1,7 +1,10 @@
 export WOUDC_API_BIND_HOST=0.0.0.0
 export WOUDC_API_BIND_PORT=5000
 export WOUDC_API_URL=http://localhost:5000/oapi
-export WOUDC_API_ES_URL=http://localhost:9200
+export WOUDC_API_ES_USERNAME=elasticsearch
+export WOUDC_API_ES_PASSWORD=<secret>
+export WOUDC_API_ES_URL=http://${WOUDC_API_ES_USERNAME}:${WOUDC_API_ES_PASSWORD}@localhost:9200
+export WOUDC_API_ES_INDEX_PREFIX=woudc_data_registry
 export WOUDC_API_OGC_SCHEMAS_LOCATION=/opt/woudc-api/schemas.opengis.net
 export PYGEOAPI_CONFIG=/path/to/woudc-api-config.yml
 export PYGEOAPI_OPENAPI=/path/to/woudc-api-openapi.yml