diff --git a/.github/workflows/testing_checks.yaml b/.github/workflows/testing_checks.yaml index b475762..571ac69 100644 --- a/.github/workflows/testing_checks.yaml +++ b/.github/workflows/testing_checks.yaml @@ -24,8 +24,7 @@ jobs: test_command: (cat action_test_files.txt | xargs python3 -m pylint --rcfile ./pylint.rc) && (find ./tests | grep '\.py' | xargs python3 -m pylint -d duplicate-code --rcfile ./pylint.rc) - app: pytest pip_installs: pytest pytest-cov - test_command: python3 -m pytest --cov=. -rpP --cov-report=xml > coverage.txt - artifacts: coverage.txt + test_command: python3 -m pytest --cov=. -rpP --cov-report=xml steps: - name: Current python version run: python3 --version || echo python3 not installed diff --git a/README.md b/README.md index 6419a25..7dd300a 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ An explanation of the command line options used follows. Be sure to read up on the [docker run](https://docs.docker.com/engine/reference/run/) command line for more information. ```bash -docker run --rm --mount "src=${PWD}/test_data,target=/mnt,type=bind" agdrone/transformer-canopycover:1.1 --working_space "/mnt" --metadata "/mnt/experiment.yaml" --citation_author "Me Myself" --citation_title "Something in the green" --citation_year "2019" --germplasm_name "Big Plant" "/mnt/rgb_1_2_E.tif" +docker run --rm --mount "src=${PWD}/test_data,target=/mnt,type=bind" agdrone/transformer-canopycover:1.1 --working_space "/mnt" --metadata "/mnt/experiment.yaml" --citation_author "Me Myself" --citation_title "Something in the green" --citation_year "2019" --species "Big Plant" "/mnt/rgb_1_2_E.tif" ``` This example command line assumes the source files are located in the `test_data` folder off the current folder. @@ -75,8 +75,13 @@ with a completely white image, make sure that 0 is returned. The reason this should be done is in order to test the extremes for image data. -Next test on these [sample plot images](https://drive.google.com/file/d/1xWRU0YgK3Y9aUy5TdRxj14gmjLlozGxo/view) and make sure -that reasonable values are returned. +Next test on sample plot images and make sure that reasonable values are returned. +The following commands can be used to retrieve the plot images: +```bash +mkdir test_data +curl https://de.cyverse.org/dl/d/4108BB75-AAA3-48E1-BBD4-E10B06CADF54/sample_plot_images.zip -o test_data/sample_plot_images.zip +unzip test_data/sample_plot_images.zip -d test_data/ +``` **Deploying the Transformer** \ Once you have used the transformer on your image data, you can upload your docker image to [Docker Hub](https://hub.docker.com) diff --git a/REVIEW.adoc b/REVIEW.adoc index 655486f..398732f 100644 --- a/REVIEW.adoc +++ b/REVIEW.adoc @@ -44,7 +44,7 @@ usage: canopycover.py [-h] [--debug] [--info] [--result [RESULT]] [--citation_author [CITATIONAUTHOR]] [--citation_title [CITATIONTITLE]] [--citation_year [CITATIONYEAR]] - [--germplasm_name [GERMPLASMNAME]] + [--species [SPECIES]] ... Canopy Cover by Plot (Percentage of Green Pixels) @@ -69,8 +69,8 @@ optional arguments: measurements --citation_year [CITATIONYEAR] year of citation to use when generating measurements - --germplasm_name [GERMPLASMNAME] - name of the germplasm associated with the canopy cover + --species [SPECIES] + name of the species associated with the canopy cover terra.stereo-rgb.canopycover version 3.0 author Chris Schnaufer schnaufer@email.arizona.edu @@ -138,7 +138,7 @@ usage: canopycover.py [-h] [--debug] [--info] [--result [RESULT]] -m METADATA [--citation_author [CITATIONAUTHOR]] [--citation_title [CITATIONTITLE]] [--citation_year [CITATIONYEAR]] - [--germplasm_name [GERMPLASMNAME]] + [--species [SPECIES]] ... canopycover.py: error: the following arguments are required: -m/--metadata ---- @@ -152,7 +152,7 @@ usage: canopycover.py [-h] [--debug] [--info] [--result [RESULT]] -m METADATA [--citation_author [CITATIONAUTHOR]] [--citation_title [CITATIONTITLE]] [--citation_year [CITATIONYEAR]] - [--germplasm_name [GERMPLASMNAME]] + [--species [SPECIES]] ... canopycover.py: error: argument -m/--metadata: can't open 'blargh': \ [Errno 2] No such file or directory: 'blargh' @@ -267,7 +267,7 @@ usage: canopycover.py [-h] [--debug] [--info] [--result [RESULT]] [--citation_author [CITATIONAUTHOR]] [--citation_title [CITATIONTITLE]] [--citation_year [CITATIONYEAR]] - [--germplasm_name [GERMPLASMNAME]] + [--species [SPECIES]] file_list [file_list ...] canopycover.py: error: the following arguments are required: file_list ---- @@ -293,7 +293,7 @@ usage: canopycover.py [-h] [--debug] [--info] [--result [RESULT]] [--citation_author [CITATIONAUTHOR]] [--citation_title [CITATIONTITLE]] [--citation_year [CITATIONYEAR]] - [--germplasm_name [GERMPLASMNAME]] + [--species [SPECIES]] ... canopycover.py: error: Invalid files: foo, bar ---- diff --git a/canopycover.py b/canopycover.py index 411e32b..a86d805 100755 --- a/canopycover.py +++ b/canopycover.py @@ -167,13 +167,13 @@ def setup_default_traits(traits: dict, args: argparse.Namespace, full_md: list) # Check metadata if full_md: for one_md in full_md: - if 'germplasmName' in one_md: - new_traits['species'] = one_md['germplasmName'] + if 'species' in one_md: + new_traits['species'] = one_md['species'] traits_modified = True # Check command line parameters - if args.germplasmName is not None: - new_traits['species'] = args.germplasmName + if args.species is not None: + new_traits['species'] = args.species traits_modified = True if args.citationAuthor is not None: new_traits['citation_author'] = args.citationAuthor @@ -198,12 +198,8 @@ def calculate_canopycover_masked(pxarray: np.ndarray) -> float: Notes: From TERRA REF canopy cover: https://github.com/terraref/extractors-stereo-rgb/tree/master/canopycover """ - # If > 75% is NoData, return a -1 ccvalue for omission later total_size = pxarray.shape[0] * pxarray.shape[1] nodata = np.count_nonzero(pxarray[:, :, 3] == 0) - nodata_ratio = nodata/float(total_size) - if nodata_ratio > 0.75: - return -1 # For masked images, all pixels with rgb>0,0,0 are considered canopy data = pxarray[pxarray[:, :, 3] == 255] @@ -249,8 +245,8 @@ def add_parameters(self, parser: argparse.ArgumentParser) -> None: parser.add_argument('--citation_year', dest="citationYear", type=str, nargs='?', help="year of citation to use when generating measurements") - parser.add_argument('--germplasm_name', dest="germplasmName", type=str, nargs='?', - help="name of the germplasm associated with the canopy cover") + parser.add_argument('--species', dest="species", type=str, nargs='?', + help="name of the species associated with the canopy cover") def check_continue(self, environment: Environment, check_md: dict, transformer_md: list, full_md: list) -> tuple: """Checks if conditions are right for continuing processing @@ -356,8 +352,6 @@ def perform_process(self, environment: Environment, check_md: dict, transformer_ else: logging.info('Adding missing alpha channel to loaded image from "%s"', one_file) image_to_use = _add_image_mask(one_file) -# mask = np.where(np.sum(pxarray, axis=0) == 0, 0, 255).astype(pxarray.dtype) -# image_to_use = np.stack((pxarray[0], pxarray[1], pxarray[2], mask)) del pxarray # Potentially free up memory logging.debug("Calculating canopy cover") @@ -389,6 +383,8 @@ def perform_process(self, environment: Environment, check_md: dict, transformer_ else: continue except Exception as ex: + if logging.getLogger().level == logging.DEBUG: + logging.exception("Exception caught while processing canopy") logging.warning("Exception caught while processing canopy cover: %s", str(ex)) logging.warning("Error generating canopy cover for '%s'", one_file) logging.warning(" plot name: '%s'", plot_name) @@ -398,7 +394,7 @@ def perform_process(self, environment: Environment, check_md: dict, transformer_ if not num_files: return {'code': -1000, 'error': "No files were processed"} if not total_plots_calculated: - return {'code': -1001, 'error': "No plots intersected with the images provided"} + return {'code': -1001, 'error': "No images were able to have their canopy cover calculated"} # Setup the metadata for returning files file_md = [] diff --git a/test_data/meta.yaml b/test_data/meta.yaml index 3367867..4073fef 100644 --- a/test_data/meta.yaml +++ b/test_data/meta.yaml @@ -3,6 +3,6 @@ pipeline: studyName: 'S7_20181011' season: 'S7_20181011' - germplasmName: Sorghum bicolor + species: Sorghum bicolor collectingSite: Maricopa observationTimeStamp: '2018-10-11T13:01:02-08:00' diff --git a/tests/test_canopy_cover.py b/tests/test_canopy_cover.py index 111481c..e8a1ed5 100644 --- a/tests/test_canopy_cover.py +++ b/tests/test_canopy_cover.py @@ -9,7 +9,9 @@ import os import random import re +import shutil import string +import tempfile from shutil import rmtree from subprocess import getstatusoutput #import pytest @@ -53,15 +55,17 @@ def test_no_args(): changed to some non-zero value to indicate a failure. """ ret_val, out = getstatusoutput(SOURCE_PATH) - assert ret_val == 0 - assert re.search('No metadata paths were specified', out) + assert ret_val == 1 + assert re.search('FileNotFoundError', out) def test_no_metadata(): """ Run with a file but no metadata""" - ret_val, out = getstatusoutput(f'{SOURCE_PATH} {INPUT1}') + temp_dir = tempfile.mkdtemp() + ret_val, out = getstatusoutput(f'{SOURCE_PATH} --working_space {temp_dir} {INPUT1}') + shutil.rmtree(temp_dir) assert ret_val == 0 - assert re.search('No metadata paths were specified', out) + assert re.search('"code": 0', out) def test_get_fields():