Merge pull request #38 from AgPipeline/develop

Merging develop to main branch - no review

.github/workflows/app_testing.json

@@ -0,0 +1,25 @@
+{
+  "BETYDB_URL": "https://terraref.org/bety",
+  "PLOT_SHAPEFILE": "/input/plot_shapes.shp",
+  "PLOT_GEOMETRY_FILE": "/output/plots.geojson",
+  "SOILMASK_SOURCE_FILE": "/input/orthomosaic.tif",
+  "SOILMASK_MASK_FILE": "orthomosaicmask.tif",
+  "SOILMASK_WORKING_FOLDER": "/output",
+  "SOILMASK_OPTIONS": "",
+  "SOILMASK_RATIO_SOURCE_FILE": "/input/orthomosaic.tif",
+  "SOILMASK_RATIO_MASK_FILE": "orthomosaicmask.tif",
+  "SOILMASK_RATIO_WORKING_FOLDER": "/output",
+  "SOILMASK_RATIO_OPTIONS": "--ratio 1.0",
+  "PLOTCLIP_SOURCE_FILE": "/input/orthomosaicmask.tif",
+  "PLOTCLIP_PLOTGEOMETRY_FILE": "/input/plots.geojson",
+  "PLOTCLIP_WORKING_FOLDER": "/output",
+  "PLOTCLIP_OPTIONS": "",
+  "FILES2JSON_SEARCH_NAME": "orthomosaicmask.tif",
+  "FILES2JSON_SEARCH_FOLDER": "/input",
+  "FILES2JSON_JSON_FILE": "/output/canopy_cover_files.json",
+  "CANOPYCOVER_OPTIONS": "",
+  "GREENNESS_INDICES_OPTIONS": "",
+  "MERGECSV_SOURCE": "/input",
+  "MERGECSV_TARGET": "/output",
+  "MERGECSV_OPTIONS": "--ignore-dirs input/outputs,test_data/"
+}

.github/workflows/check_canopycover_app.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Define some counts that we expect
+EXPECTED_NUM_CANOPYCOVER_CSV=57
+
+# What folder are we looking in for outputs
+if [[ ! "${1}" == "" ]]; then
+  TARGET_FOLDER="${1}"
+else
+  TARGET_FOLDER="./outputs"
+fi
+
+# Check the expected number of output files
+# shellcheck disable=SC2207
+EXPECTED_CSV=($(find "${TARGET_FOLDER}/" -type f | grep 'canopycover\.csv' | grep -v 'test_data'))
+if [[ "${#EXPECTED_CSV[@]}" == "${EXPECTED_NUM_CANOPYCOVER_CSV}" ]]; then
+  echo "Found expected number of canopycover.csv files: ${EXPECTED_NUM_CANOPYCOVER_CSV}"
+else
+  echo "Expected ${EXPECTED_NUM_CANOPYCOVER_CSV} canopycover.csv files but found ${#EXPECTED_CSV[@]}"
+  for i in $(seq 0 $((${#EXPECTED_CSV[@]} - 1))); do
+    echo "$((i + 1)): ${EXPECTED_CSV[$i]}"
+  done
+  exit 20
+fi

.github/workflows/check_csv_details.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# Check for a filename override
+FILENAME="canopycover.csv"
+if [[ ! "${1}" == "" ]]; then
+  FILENAME="${1}"
+fi
+echo "Checking files named ${FILENAME}"
+
+TRUTH_FOLDER="test_data"
+if [[ ! "${2}" == "" ]]; then
+  TRUTH_FOLDER="${2}"
+fi
+echo "Searching folder ${TRUTH_FOLDER}"
+
+COMPARE_FOLDER="outputs"
+if [[ ! "${3}" == "" ]]; then
+  COMPARE_FOLDER="${3}"
+fi
+echo "Comparison folder ${COMPARE_FOLDER}"
+
+PRIMARY_KEY_COLUMNS="4"
+if [[ ! "${4}" == "" ]]; then
+  PRIMARY_KEY_COLUMNS="${4}"
+fi
+echo "Primary key columns ${PRIMARY_KEY_COLUMNS}"
+
+COMPARE_COLUMNS="1"
+if [[ ! "${5}" == "" ]]; then
+  COMPARE_COLUMNS="${5}"
+fi
+echo "Columns to compare ${COMPARE_COLUMNS}"
+
+TRUTH_FOLDER_LEN="${#TRUTH_FOLDER}"
+# shellcheck disable=SC2207
+FOLDER_LIST=($(find "${TRUTH_FOLDER}/" -maxdepth 1 -type d))
+
+# Get the line count of differences between the files
+for subfolder in "${FOLDER_LIST[@]}"; do
+  WORKING_FOLDER="${subfolder:${TRUTH_FOLDER_LEN}}"
+
+  if [[ "${WORKING_FOLDER}" == "" ]]; then
+    continue
+  fi
+  if [[ "${WORKING_FOLDER}" == "/" ]]; then
+    continue
+  fi
+
+  # shellcheck disable=SC2126
+  DIFF_RESULT="$(./csvdiff/csvdiff "${TRUTH_FOLDER}/${WORKING_FOLDER}/${FILENAME}" "${COMPARE_FOLDER}/${WORKING_FOLDER}/${FILENAME}" --columns 1 --primary-key "${PRIMARY_KEY_COLUMNS}" --format rowmark 2>&1 | grep -A 5 'Rows:' | wc -l | tr -d '[:space:]')"
+
+  if [[ "${DIFF_RESULT}" == "1" ]]; then
+    echo "No differences: ${TRUTH_FOLDER}/${WORKING_FOLDER} for file ${FILENAME}"
+  else
+    echo "Error: folder ${TRUTH_FOLDER}/${WORKING_FOLDER} file ${FILENAME} doesn't match"
+    echo "Comparison folder listing"
+    ls -l "${COMPARE_FOLDER}/${WORKING_FOLDER}"
+    echo "Generated file contents"
+    cat "${TRUTH_FOLDER}/${WORKING_FOLDER}/${FILENAME}"
+    echo "Comparison file contents"
+    cat "${COMPARE_FOLDER}/${WORKING_FOLDER}/${FILENAME}"
+    echo "CSV differences result"
+    ./csvdiff/csvdiff "${TRUTH_FOLDER}/${WORKING_FOLDER}/${FILENAME}" "${COMPARE_FOLDER}/${WORKING_FOLDER}/${FILENAME}" --columns 1 --primary-key 4 --format rowmark
+    exit 10
+  fi
+done

.github/workflows/check_greenness_indices_app.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Define some counts that we expect
+EXPECTED_NUM_GREENNESS_CSV=57
+
+# What folder are we looking in for outputs
+if [[ ! "${1}" == "" ]]; then
+  TARGET_FOLDER="${1}"
+else
+  TARGET_FOLDER="./outputs"
+fi
+
+# Check the expected number of output files
+# shellcheck disable=SC2207
+EXPECTED_CSV=($(find "${TARGET_FOLDER}/" -type f | grep 'rgb_plot\.csv' | grep -v 'test_data'))
+if [[ "${#EXPECTED_CSV[@]}" == "${EXPECTED_NUM_GREENNESS_CSV}" ]]; then
+  echo "Found expected number of rgb_plot.csv files: ${EXPECTED_NUM_GREENNESS_CSV}"
+else
+  echo "Expected ${EXPECTED_NUM_GREENNESS_CSV} rgb_plot.csv files but found ${#EXPECTED_CSV[@]}"
+  for i in $(seq 0 $((${#EXPECTED_CSV[@]} - 1))); do
+    echo "$((i + 1)): ${EXPECTED_CSV[$i]}"
+  done
+  exit 20
+fi

.github/workflows/check_plotclip_app.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Define some counts that we expect (the number of sub-folders plus the top folder)
+EXPECTED_NUM_FOLDERS=58
+EXPECTED_NUM_CLIPS=57
+# Make file to look for: should match what's in the configuration JSON file
+MASK_FILE_NAME="orthomosaicmask.tif"
+MASK_FILE_NAME_GREP="orthomosaicmask\\.tif"
+
+# What folder are we looking in for outputs
+if [[ ! "${1}" == "" ]]; then
+  TARGET_FOLDER="${1}"
+else
+  TARGET_FOLDER="./outputs"
+fi
+
+# Get all the folders and check the count
+# shellcheck disable=SC2207
+FOLDER_LIST=($(find "${TARGET_FOLDER}/" -maxdepth 1 -type d))
+if [[ "${#FOLDER_LIST[@]}" == "${EXPECTED_NUM_FOLDERS}" ]]; then
+  echo "Found expected number of folders: ${EXPECTED_NUM_FOLDERS}"
+else
+  echo "Expected ${EXPECTED_NUM_FOLDERS} folders and found ${#FOLDER_LIST[@]}"
+  for i in $(seq 0 $((${#FOLDER_LIST[@]} - 1))); do
+    echo "$((i + 1)): ${FOLDER_LIST[$i]}"
+  done
+  exit 10
+fi
+
+# Check the expected number of image mask files
+# shellcheck disable=SC2207
+EXPECTED_CLIPS=($(find "${TARGET_FOLDER}/" -type f | grep "${MASK_FILE_NAME_GREP}"))
+if [[ "${#EXPECTED_CLIPS[@]}" == "${EXPECTED_NUM_CLIPS}" ]]; then
+  echo "Found expected number of ${MASK_FILE_NAME} files: ${EXPECTED_NUM_CLIPS}"
+else
+  echo "Expected ${EXPECTED_NUM_CLIPS} ${MASK_FILE_NAME} files but found ${#EXPECTED_CLIPS[@]}"
+  for i in $(seq 0 $((${#EXPECTED_CLIPS[@]} - 1))); do
+    echo "$((i + 1)): ${EXPECTED_CLIPS[$i]}"
+  done
+  exit 30
+fi

.github/workflows/check_plotclip_details.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+"""Python script for converting BETYdb plot outlines to GeoJSON format
+"""
+
+import argparse
+import os
+import subprocess
+
+# The default file extension to look for
+DEFAULT_FILE_EXT = '.tif'
+
+
+def get_arguments() -> argparse.Namespace:
+    """Adds arguments to the command line parser
+    Return:
+        Returns the parsed arguments
+    """
+    parser = argparse.ArgumentParser(description="Checking plotclip results",
+                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('-e', '--file-ext',
+                        help='the case-insensitive extension of the image files to find; defaults to ' + DEFAULT_FILE_EXT,
+                        type=str, default=DEFAULT_FILE_EXT)
+    parser.add_argument('-n', '--no-recurse', default=False, action='store_true',
+                        help='do not check subfolders for files; default is to check subfolders; implies -t')
+    parser.add_argument('-t', '--top-folder', default=False, action='store_true',
+                        help='check the top level folders for matching files; default is to skip top folder checks')
+    parser.add_argument('truth_folder', type=str,
+                        help='path to the top level folder containing the files to use for comparison')
+    parser.add_argument('check_folder', type=str,
+                        help='path to the top level folder containing the file to check')
+
+    args = parser.parse_args()
+
+    return args
+
+
+def compare_file_contents(source_file: str, compare_file: str) -> bool:
+    """Compares the source file against the comparison file for differences
+    Arguments:
+        source_file: the file that is checked against
+        compare_file: the file that is compared against the source_file
+    Returns:
+        Returns True if the files are considered the same, and False if they are not
+    """
+    cmd = ['diff', '--brief', source_file, compare_file]
+    res = subprocess.run(cmd, capture_output=True, check=True)
+    return len(res.stdout) == 0
+
+
+def find_compare_files(truth_folder: str, compare_folder: str, file_ext: str, check_subfolders: bool = True) -> None:
+    """Searches the folders for files with the matching extension and compares the contents
+    Arguments:
+        truth_folder: the folder containing the copies that are the truth
+        compare_folder: the folder to compare the truth files against
+        file_ext: the file extension to look for
+        check_subfolders: also look for files in the subfolders off the truth_folder
+    Exceptions:
+        Throws a RuntimeError when an image file doesn't match the expected contents, or isn't found in
+        the compare folder
+    Notes:
+        The truth_folder is searched for matching file extensions, the compare_folder is not. This means
+        that additional matching files in the compare_folder are ignored
+    """
+    # Initialize folders to check with the top-level folder
+    check_folders = [""]
+    # Make sure the extension is in the correct format
+    check_ext = file_ext.lower() if file_ext[0] == '.' else '.' + file_ext.lower()
+
+    # While we have folders to process
+    while len(check_folders) > 0:
+        cur_subpath = check_folders.pop(0)
+        cur_truth = os.path.join(truth_folder, cur_subpath)
+        cur_compare = os.path.join(compare_folder, cur_subpath)
+
+        # Check truth folder for matching files and sub-folders
+        for one_file in os.listdir(cur_truth):
+            if os.path.splitext(one_file)[1].lower() == check_ext:
+                if not os.path.exists(os.path.join(cur_compare, one_file)):
+                    raise RuntimeError('Unable to find expected file "%s"' % os.path.join(cur_subpath, one_file))
+                if not compare_file_contents(os.path.join(cur_truth, one_file), os.path.join(cur_compare, one_file)):
+                    raise RuntimeError('File content mismatch "%s"' % os.path.join(cur_subpath, one_file))
+            elif check_subfolders and os.path.isdir(os.path.join(cur_truth, one_file)):
+                # Queue the subfolder for later comparison
+                check_folders.append(os.path.join(cur_subpath, one_file))
+
+
+def check_details() -> None:
+    """Checks the plot clipping details to ensure that the contents of the clipped plots
+       match what's expected
+    Exceptions:
+        A RuntimeError is thrown when an image file doesn't match the expected contents, or there's
+        a folder content mismatch
+    Notes:
+        The truth_folder is searched for matching file extensions, the compare_folder is not. This means
+        that additional matching files in the compare_folder are ignored
+    """
+    # Get the command line parameters
+    args = get_arguments()
+
+    # Check the folder parameters
+    bad_folders = []
+    if not os.path.exists(args.truth_folder):
+        bad_folders.append(args.truth_folder)
+    if not os.path.exists(args.check_folder):
+        bad_folders.append(args.check_folder)
+    if len(bad_folders) > 0:
+        raise ValueError("Please correct folder paths and try again: %s" %
+                         ",".join(['"' + folder + '"' for folder in bad_folders]))
+
+    # Start searching folders
+    if args.no_recurse or args.top_folder:
+        find_compare_files(args.truth_folder, args.check_folder, args.file_ext, False)
+    else:
+        for one_file in os.listdir(args.truth_folder):
+            cur_path = os.path.join(args.truth_folder, one_file)
+            if os.path.isdir(cur_path):
+                test_path = os.path.join(args.check_folder, one_file)
+                find_compare_files(cur_path, test_path, args.file_ext)
+
+
+if __name__ == "__main__":
+    check_details()