Merge pull request #108 from bird-house/add_geospatial

Add geospatial process and GIS pip recipe

CHANGES.rst

@@ -1,6 +1,16 @@
 Changes
 *******
 
+0.13.0 (unreleased)
+===================
+
+Changes:
+
+* Added Geospatial process.
+* Added a new recipe for installing Emu with GIS libraries (`pip install eum[gis]`).
+* Added example geospatial data (raster image courtesy of USGS: `Mars MGS MOLA DEM 463m v2`).
+* Refined Ultimate Question process to be truer to the source literature.
+
 0.12.0 (2020-10-07)
 ===================
 

emu/data/Olympus_Mons.geojson

@@ -0,0 +1,8 @@
+{
+"type": "FeatureCollection",
+"name": "olympus_mons",
+"crs": { "type": "name", "properties": { "name": "urn:ogc:def:crs:OGC:1.3:CRS84" } },
+"features": [
+{ "type": "Feature", "properties": { "id": 0 }, "geometry": { "type": "MultiPolygon", "coordinates": [ [ [ [ 133.821082132244499, -182.172079007781065 ], [ 119.87448803812417, -187.178548682593487 ], [ 117.371253200717959, -200.052327846396878 ], [ 121.304907945213429, -211.138082126338645 ], [ 137.754736876739969, -211.495687103110981 ], [ 147.767676226364813, -206.131612451526223 ], [ 146.337256319275525, -188.608968589682775 ], [ 133.821082132244499, -182.172079007781065 ] ] ] ] } }
+]
+}

emu/processes/__init__.py

@@ -18,6 +18,7 @@
 from .wps_dry_run import SimpleDryRun
 from .wps_ncml import NcMLAgg
 from .wps_translation import Translation
+from .wps_geodata import GeoData
 
 
 processes = [
@@ -41,4 +42,5 @@
     SimpleDryRun(),
     NcMLAgg(),
     Translation(),
+    GeoData(),
 ]

emu/processes/wps_esgf.py

@@ -1,11 +1,9 @@
-from pywps import Process
-from pywps import LiteralInput, LiteralOutput
-from pywps import ComplexInput
-from pywps import Format
+import logging
+
+from pywps import ComplexInput, Format, LiteralInput, LiteralOutput, Process
 from pywps.app.Common import Metadata
 from pywps.ext_autodoc import MetadataUrl
 
-import logging
 LOGGER = logging.getLogger("PYWPS")
 
 
@@ -69,7 +67,7 @@ def _handler(request, response):
                 datasets.append(dataset.data)
         if not datasets:
             raise Exception("You need to provide at least one dataset.")
-        response.outputs['output'].data = 'Number of datasets = {}'.format(len(datasets))
+        response.outputs['output'].data = f'Number of datasets = {len(datasets)}'
 
         response.update_status('PyWPS Process completed.', 100)
         return response

emu/processes/wps_geodata.py

@@ -0,0 +1,61 @@
+"""
+Process using an application/geo+json and 'image/tiff; application=geotiff' output. Used to test UI interactions.
+
+Author: Trevor James Smith
+"""
+from pathlib import Path
+from pywps import Process, ComplexOutput
+from pywps import FORMATS
+import logging
+
+
+LOGGER = logging.getLogger("PYWPS")
+
+__all__ = ["GeoData"]
+
+DATA_DIR = Path(__file__).parent.parent.joinpath('data')
+
+
+class GeoData(Process):
+    def __init__(self):
+        inputs = list()
+        outputs = [
+            ComplexOutput(
+                "raster",
+                "DEM of region in GeoTIFF format.",
+                abstract="Elevation data for Olympus Martian region in byte range.",
+                as_reference=True,
+                supported_formats=[FORMATS.GEOTIFF],
+            ),
+            ComplexOutput(
+                "vector",
+                "Region definition in GeoJSON format",
+                abstract="Rough vector polygon of the Olympus Mons plateau.",
+                as_reference=True,
+                supported_formats=[FORMATS.GEOJSON],
+            ),
+        ]
+
+        super(GeoData, self).__init__(
+            self._handler,
+            identifier="geodata",
+            title="deliver vector and raster data",
+            abstract="Return an example raster and vector dataset",
+            version="1.0",
+            inputs=inputs,
+            outputs=outputs,
+            store_supported=True,
+            status_supported=True,
+        )
+
+    @staticmethod
+    def _handler(request, response):
+        response.update_status("PyWPS Process started.", 0)
+
+        response.outputs['vector'].file = DATA_DIR / "Olympus_Mons.geojson"
+
+        response.outputs['raster'].file = DATA_DIR / "Olympus.tif"
+
+        response.update_status("PyWPS Process completed.", 100)
+
+        return response

emu/processes/wps_output_formats.py

@@ -3,11 +3,11 @@
 
 Author: David Huard
 """
+import logging
 import os
-from pywps import Process, ComplexOutput
-from pywps import FORMATS
 
-import logging
+from pywps import FORMATS, ComplexOutput, Process
+
 LOGGER = logging.getLogger("PYWPS")
 
 # TODO: can be replaced by eggshell function.
@@ -16,7 +16,7 @@
 
 class OutputFormats(Process):
     def __init__(self):
-        inputs = []
+        inputs = list()
         outputs = [
             ComplexOutput('netcdf', 'netCDF dummy output file.',
                           abstract="A very small test netCDF file. ",

emu/processes/wps_ultimate_question.py

@@ -12,7 +12,7 @@ def __init__(self):
             identifier='ultimate_question',
             version='2.0',
             title='Answer to the ultimate question',
-            abstract='This process gives the answer to the ultimate question of "What is the meaning of life?"',
+            abstract='This process gives the answer to the ultimate question of life, the universe, and everything.',
             profile='',
             metadata=[Metadata('Ultimate Question'), Metadata('What is the meaning of life')],
             inputs=inputs,
@@ -24,9 +24,11 @@ def __init__(self):
     @staticmethod
     def _handler(request, response):
         import time
+        sleep_delay = .1
         response.update_status('PyWPS Process started.', 0)
+        time.sleep(sleep_delay)
 
-        sleep_delay = .1
+        response.update_status("Contacting the Deep Thought supercomputer.", 10)
         time.sleep(sleep_delay)
         response.update_status('Thinking...', 20)
         time.sleep(sleep_delay)

emu/processes/wps_wordcounter.py

@@ -1,13 +1,11 @@
 import json
+import logging
 import re
-
 from collections import Counter
 
-from pywps import Process
-from pywps import ComplexInput, ComplexOutput, FORMATS
+from pywps import FORMATS, ComplexInput, ComplexOutput, Process
 from pywps.ext_autodoc import MetadataUrl
 
-import logging
 LOGGER = logging.getLogger("PYWPS")
 
 

environment-docs.yml

@@ -4,7 +4,7 @@ channels:
 - conda-forge
 - defaults
 dependencies:
-- pywps>=4.2.7
+- pywps=4.4.0
 - sphinx
 - nbsphinx
 - ipython

environment.yml

@@ -4,8 +4,8 @@ channels:
 - defaults
 dependencies:
 - pip
-- python=3.6
-- pywps>=4.2.6,<4.3
+- python>=3.6
+- pywps>=4.4
 - jinja2
 - click
 - psutil

requirements.txt

@@ -1,4 +1,4 @@
-pywps>=4.2.6<4.3
+pywps>=4.4.0
 jinja2
 click
 psutil

setup.py

@@ -10,14 +10,15 @@
 here = os.path.abspath(os.path.dirname(__file__))
 README = open(os.path.join(here, 'README.rst')).read()
 CHANGES = open(os.path.join(here, 'CHANGES.rst')).read()
-REQUIRES_PYTHON = ">=2.7.0"
+REQUIRES_PYTHON = ">=3.6.0"
 
 about = {}
 with open(os.path.join(here, 'emu', '__version__.py'), 'r') as f:
     exec(f.read(), about)
 
 reqs = [line.strip() for line in open('requirements.txt')]
 dev_reqs = [line.strip() for line in open('requirements_dev.txt')]
+extra_reqs = [line.strip() for line in open("requirements_extra.txt")]
 
 classifiers = [
     'Development Status :: 3 - Alpha',
@@ -27,10 +28,11 @@
     'Operating System :: POSIX',
     'Programming Language :: Python',
     'Natural Language :: English',
-    "Programming Language :: Python :: 2",
-    'Programming Language :: Python :: 2.7',
     'Programming Language :: Python :: 3',
     'Programming Language :: Python :: 3.6',
+    'Programming Language :: Python :: 3.7',
+    'Programming Language :: Python :: 3.8',
+    'Programming Language :: Python :: 3.9',
     'Topic :: Scientific/Engineering :: Atmospheric Science',
     'License :: OSI Approved :: Apache Software License',
 ]
@@ -51,6 +53,7 @@
       include_package_data=True,
       install_requires=reqs,
       extras_require={
+          "extra:": extra_reqs,
           "dev": dev_reqs,              # pip install ".[dev]"
       },
       entry_points={

tests/common.py

@@ -37,15 +37,15 @@ def get_output(doc):
         [identifier_el] = xpath_ns(output_el, './ows:Identifier')
 
         lit_el = xpath_ns(output_el, './wps:Data/wps:LiteralData')
-        if lit_el != []:
+        if lit_el:
             output[identifier_el.text] = lit_el[0].text
 
         ref_el = xpath_ns(output_el, './wps:Reference')
-        if ref_el != []:
+        if ref_el:
             output[identifier_el.text] = ref_el[0].attrib['href']
 
         data_el = xpath_ns(output_el, './wps:Data/wps:ComplexData')
-        if data_el != []:
+        if data_el:
             output[identifier_el.text] = data_el[0].text
 
     return output

tests/test_wps_bbox.py

@@ -3,16 +3,19 @@
 from pywps import Service
 from pywps.tests import assert_response_success
 
-from .common import client_for
+from .common import client_for, get_output
 from emu.processes.wps_bbox import Box
 
 
-@pytest.mark.skip(reason="no way of currently testing this")
+@pytest.mark.xfail(reason="bbox doesn't seem to be joined to response output")
 def test_wps_bbox():
     client = client_for(Service(processes=[Box()]))
     datainputs = "bbox=101,42,110,46"
     resp = client.get(
         service='WPS', request='Execute', version='1.0.0',
         identifier='bbox',
         datainputs=datainputs)
+
     assert_response_success(resp)
+    out = get_output(resp.xml)
+    assert out["bbox"] == [101, 42, 110, 46]

tests/test_wps_caps.py

@@ -18,6 +18,7 @@ def test_wps_caps():
         'chomsky',
         'dummyprocess',
         'esgf_demo',
+        'geodata',
         'hello',
         'inout',
         'multiple_outputs',

tests/test_wps_error.py

@@ -8,8 +8,8 @@ def test_wps_error():
     client = client_for(Service(processes=[ShowError()]))
     datainputs = "message=tomorrow-is-another-day;nice=true"
     client.get(
-        "?service=WPS&request=Execute&version=1.0.0&identifier=hello&datainputs={}".format(
-            datainputs))
+        f"?service=WPS&request=Execute&version=1.0.0&identifier=hello&datainputs={datainputs}"
+    )
     # TODO: parse ows:Exception
     # assert_response_success(resp)
     # assert get_output(resp.xml) == {'output': "Hello LovelySugarBird"}

tests/test_wps_geo.py

@@ -0,0 +1,17 @@
+from pywps import Service
+from pywps.tests import assert_response_success
+from emu.processes.wps_geodata import GeoData
+from .common import client_for, CFG_FILE, get_output
+
+
+class TestGeoData:
+    def test_wps_geodata(self):
+
+        client = client_for(Service(processes=[GeoData()], cfgfiles=CFG_FILE))
+
+        resp = client.get(
+            service='wps', request='execute', version='1.0.0',
+            identifier='geodata', asobj=True)
+
+        assert_response_success(resp)
+        assert set(get_output(resp.xml).keys()) == {"vector", "raster"}

tests/test_wps_hello.py

@@ -11,7 +11,7 @@ def test_wps_hello():
     client = client_for(Service(processes=[SayHello()]))
     datainputs = "name=LovelySugarBird"
     resp = client.get(
-        "?service=WPS&request=Execute&version=1.0.0&identifier=hello&datainputs={}".format(
-            datainputs))
+        f"?service=WPS&request=Execute&version=1.0.0&identifier=hello&datainputs={datainputs}"
+    )
     assert_response_success(resp)
     assert get_output(resp.xml) == {'output': "Hello LovelySugarBird"}

tests/test_wps_inout.py

@@ -1,5 +1,3 @@
-import pytest
-import os
 from pywps import Service
 from pywps.tests import assert_response_success
 

tests/test_wps_multiple_outputs.py

@@ -1,5 +1,4 @@
 import pytest
-from lxml import etree
 from pywps import Service
 from pywps.tests import assert_response_success
 
@@ -10,7 +9,7 @@
 @pytest.fixture
 def resp():
     client = client_for(Service(processes=[MultipleOutputs()], cfgfiles=CFG_FILE))
-    datainputs = 'count={}'.format(5)
+    datainputs = 'count=5'
     response = client.get(
         service="WPS", request="Execute", version="1.0.0", identifier="multiple_outputs",
         datainputs=datainputs)

tests/test_wps_ncmeta.py

@@ -5,10 +5,9 @@
 
 from .common import client_for, resource_file
 from emu.processes.wps_ncmeta import NCMeta
-import owslib.wps
 
 OPENDAP_URL = 'http://test.opendap.org:80/opendap/netcdf/examples/sresa1b_ncar_ccsm3_0_run1_200001.nc'
-NC_URL = 'http://test.opendap.org:80/opendap/netcdf/examples/sresa1b_ncar_ccsm3_0_run1_200001.nc.nc4'
+NC_URL = "http://test.opendap.org/opendap/nc4_test_files/nc4_nc_classic_comp.nc.nc4"
 NC_FILE_URL = "file://{}".format(resource_file('test.nc'))
 
 

tests/test_wps_nonpyid.py

@@ -10,7 +10,7 @@
 def test_wps_nonpyid():
     d = {'a': 1}
     client = client_for(Service(processes=[NonPyID()]))
-    datainputs = "input 1=10;input-2={}".format(json.dumps(d))
+    datainputs = f"input 1=10;input-2={json.dumps(d)}"
     resp = client.get(
         service='WPS', request='Execute', version='1.0.0',
         identifier='non.py-id',