Merge branch 'signalstrength'

datasets/bathy/CMakeLists.txt

@@ -44,6 +44,7 @@ target_sources (slideruleLib
         ${CMAKE_CURRENT_LIST_DIR}/package/BathyQtreesClassifier.cpp
         ${CMAKE_CURRENT_LIST_DIR}/package/BathyRefractionCorrector.cpp
         ${CMAKE_CURRENT_LIST_DIR}/package/BathySeaSurfaceFinder.cpp
+        ${CMAKE_CURRENT_LIST_DIR}/package/BathySignalStrength.cpp
         ${CMAKE_CURRENT_LIST_DIR}/package/BathyUncertaintyCalculator.cpp
         ${CMAKE_CURRENT_LIST_DIR}/package/BathyViewer.cpp
         $<$<CONFIG:Debug>:${CMAKE_CURRENT_LIST_DIR}/package/UT_BathyRefractionCorrector.cpp>
@@ -68,6 +69,7 @@ install (
         ${CMAKE_CURRENT_LIST_DIR}/package/BathyQtreesClassifier.h
         ${CMAKE_CURRENT_LIST_DIR}/package/BathyRefractionCorrector.h
         ${CMAKE_CURRENT_LIST_DIR}/package/BathySeaSurfaceFinder.h
+        ${CMAKE_CURRENT_LIST_DIR}/package/BathySignalStrength.h
         ${CMAKE_CURRENT_LIST_DIR}/package/BathyUncertaintyCalculator.h
         ${CMAKE_CURRENT_LIST_DIR}/package/BathyViewer.h
         $<$<CONFIG:Debug>:${CMAKE_CURRENT_LIST_DIR}/package/UT_BathyRefractionCorrector.h>

datasets/bathy/docker/oceaneyes/atl24_iso_xml_template.txt

@@ -74,7 +74,7 @@
                   <gmd:identifier>
                     <gmd:MD_Identifier>
                       <gmd:code>
-                        <gco:CharacterString>006</gco:CharacterString>
+                        <gco:CharacterString>001</gco:CharacterString>
                       </gmd:code>
                       <gmd:description>
                         <gco:CharacterString>The ECS Version ID</gco:CharacterString>
@@ -108,7 +108,7 @@
                       </gmd:title>
                       <gmd:date gco:nilReason="unknown"/>
                       <gmd:edition>
-                        <gco:CharacterString>006</gco:CharacterString>
+                        <gco:CharacterString>001</gco:CharacterString>
                       </gmd:edition>
                     </gmd:CI_Citation>
                   </gmd:aggregateDataSetName>

datasets/bathy/docker/oceaneyes/openoceans/openoceans.py

@@ -33,31 +33,31 @@
 # ----------------------------------------------------------------------------
 #
 # Copyright (c) 2022, Jonathan Markel/UT Austin.
-# 
-# Redistribution and use in source and binary forms, with or without 
+#
+# Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are met:
-# 
-# Redistributions of source code must retain the above copyright notice, 
+#
+# Redistributions of source code must retain the above copyright notice,
 # this list of conditions and the following disclaimer.
 #
-# Redistributions in binary form must reproduce the above copyright notice, 
-# this list of conditions and the following disclaimer in the documentation 
+# Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
 # and/or other materials provided with the distribution.
 #
-# Neither the name of the copyright holder nor the names of its 
-# contributors may be used to endorse or promote products derived from this 
-# software without specific prior written permission. 
-# 
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 
-# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 
-# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 
-# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 
-# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 
-# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 
-# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
+# Neither the name of the copyright holder nor the names of its
+# contributors may be used to endorse or promote products derived from this
+# software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR '
-# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 
+# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 # ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import pandas as pd
@@ -93,14 +93,14 @@ def OpenOceans( df,*,
                 range_z,
                 verbose,
                 chunk_size):
-    
+
     ph_data = pd.DataFrame()
 
     ph_data["photon_index"] = df["index_ph"] # integer position of photon within the h5 file
     ph_data["x_ph"] = df["x_atc"] - min(df["x_atc"]) # total along track distance, normalized to the minimum value
     ph_data["z_ph"] = df["geoid_corr_h"] # geoid-corrected photon height
 
-    ph_data["quality_ph"] = df["quality_ph"] # modelling_parallel.py needs thiss
+    ph_data["quality_ph"] = df["quality_ph"] # modelling_parallel.py needs this
 
     ph_data["classification"] = NO_VALUE
     ph_data["conf_background"] = NO_VALUE
@@ -116,7 +116,7 @@ def OpenOceans( df,*,
     beam_strength = {
         1: "strong",
         2: "weak",
-        3: "strong", 
+        3: "strong",
         4: "weak",
         5: "strong",
         6: "weak"

datasets/bathy/docker/oceaneyes/runner.py

@@ -325,13 +325,18 @@ def runClassifier(classifier, classifier_func, num_processes=6):
 df = pd.concat([beam_table[beam] for beam in beam_list])
 print("Concatenated data frames into a single data frame")
 
-# set processing flags
+# set confidence in processing flags
+df["processing_flags"] = df["processing_flags"] + ((df["confidence"] * 256).astype(np.uint8) * 256)
+
+## set individual classifier bits in processing flags
 df["processing_flags"] = df["processing_flags"] + \
-                        ((df["confidence"] * 256).astype(np.uint8) * 256) + \
-                        ((df["cshelph"] == 40) * 2**28) + \
-                        ((df["medianfilter"] == 40) * 2**27) + \
-                        ((df["bathypathfinder"] == 40) * 2**29) + \
-                        ((df["pointnet"] == 40) * 2**30)
+                       ((df["qtrees"]          == 40) * 2**24) + \
+                       ((df["coastnet"]        == 40) * 2**25) + \
+                       ((df["openoceanspp"]    == 40) * 2**26) + \
+                       ((df["medianfilter"]    == 40) * 2**27) + \
+                       ((df["cshelph"]         == 40) * 2**28) + \
+                       ((df["bathypathfinder"] == 40) * 2**29) + \
+                       ((df["pointnet"]        == 40) * 2**30)
 
 # apply subaqueous corrections
 corrections_start_time = time.time()

datasets/bathy/endpoints/atl24g.lua

@@ -205,7 +205,6 @@ local granule = (parms["output"]["format"] == "h5") and bathy.granule(parms, atl
 local kd490 = bathy.kd(parms, viirs_filename)
 local refraction = bathy.refraction(parms)
 local uncertainty = bathy.uncertainty(parms, kd490)
-local seasurface = parms["find_sea_surface"] and bathy.seasurface(parms) or nil
 local qtrees = parms:classifier(bathy.QTREES) and bathy.qtrees(parms) or nil
 local coastnet = parms:classifier(bathy.COASTNET) and bathy.coastnet(parms) or nil
 local openoceanspp = parms:classifier(bathy.OPENOCEANSPP) and bathy.openoceanspp(parms) or nil
@@ -220,7 +219,6 @@ for _, beam in ipairs(parms["beams"]) do
     if not dataframes[beam] then
         userlog:alert(core.CRITICAL, core.RTE_ERROR, string.format("request <%s> on %s failed to create bathy dataframe for beam %s", rspq, resource, beam))
     else
-        dataframes[beam]:run(seasurface)
         dataframes[beam]:run(qtrees)
         dataframes[beam]:run(coastnet)
         dataframes[beam]:run(openoceanspp)
@@ -312,7 +310,6 @@ kd490:destroy()
 -------------------------------------------------------
 -- get profiles
 -------------------------------------------------------
-profile["seasurface"] = seasurface and seasurface:runtime() or 0.0
 profile["refraction"] = refraction and refraction:runtime() or 0.0
 profile["uncertainty"] = uncertainty and uncertainty:runtime() or 0.0
 profile["qtrees"] = qtrees and qtrees:runtime() or 0.0
@@ -321,6 +318,14 @@ profile["openoceanspp"] = openoceanspp and openoceanspp:runtime() or 0.0
 profile["duration"] = (time.gps() - start_time) / 1000.0
 userlog:alert(core.INFO, core.RTE_INFO, string.format("request <%s> ATL24 runtime at %0.3f", rspq, profile["duration"]))
 
+-------------------------------------------------------
+-- set atl24 output filename
+-------------------------------------------------------
+local atl24_filename = parms["output"]["path"]
+local pos_last_delim = string.reverse(atl24_filename):find("/") or -(#atl24_filename + 2)
+outputs["atl24_filename"] = string.sub(atl24_filename, #atl24_filename - pos_last_delim + 2)
+userlog:alert(core.INFO, core.RTE_INFO, string.format("request <%s> generating file %s", rspq, outputs["atl24_filename"]))
+
 -------------------------------------------------------
 -- set additional outputs
 -------------------------------------------------------
@@ -329,7 +334,6 @@ outputs["format"] = parms["output"]["format"]
 outputs["filename"] = crenv.container_sandbox_mount.."/"..tmp_filename
 outputs["ensemble"] = parms["ensemble"] or {ensemble_model_filename=string.format("%s/%s", cre.HOST_DIRECTORY, bathy.ENSEMBLE_MODEL)}
 outputs["iso_xml_filename"] = crenv.container_sandbox_mount.."/"..tmp_filename..".iso.xml"
-outputs["atl24_filename"] = string.gsub(parms["resource"], "ATL03", "ATL24")
 outputs["latch"] = latch
 
 -------------------------------------------------------

datasets/bathy/package/BathyCoastnetClassifier.cpp

@@ -111,7 +111,6 @@ bool BathyCoastnetClassifier::run (GeoDataFrame* dataframe)
         FieldColumn<int8_t>& class_ph = *dynamic_cast<FieldColumn<int8_t>*>(dataframe->getColumnData("class_ph"));
         FieldColumn<float>& surface_h = *dynamic_cast<FieldColumn<float>*>(dataframe->getColumnData("surface_h"));
         FieldColumn<FieldArray<int8_t, BathyFields::NUM_CLASSIFIERS>>& predictions = *dynamic_cast<FieldColumn<FieldArray<int8_t, BathyFields::NUM_CLASSIFIERS>>*>(dataframe->getColumnData("predictions"));
-        FieldColumn<uint32_t>& processing_flags = *dynamic_cast<FieldColumn<uint32_t>*>(dataframe->getColumnData("processing_flags"));
 
         // Preallocate samples and predictions vector
         const size_t number_of_samples = dataframe->length();
@@ -151,10 +150,6 @@ bool BathyCoastnetClassifier::run (GeoDataFrame* dataframe)
             if(args.setSurface) surface_h[i] = results[i].surface_elevation;
             if(args.setClass) class_ph[i] = results[i].prediction;
             predictions[i][BathyFields::COASTNET] = results[i].prediction;
-            if(results[i].prediction == BathyFields::BATHYMETRY)
-            {
-                processing_flags[i] = processing_flags[i] | BathyFields::BATHY_COASTNET;
-            }
         }
     }
     catch (const std::exception &e)

datasets/bathy/package/BathyDataFrame.cpp

@@ -122,7 +122,6 @@ BathyDataFrame::BathyDataFrame (lua_State* L, const char* beam_str, BathyFields*
         {"sigma_thu",           &sigma_thu},
         {"sigma_tvu",           &sigma_tvu},
         {"processing_flags",    &processing_flags},
-        {"yapc_score",          &yapc_score},
         {"max_signal_conf",     &max_signal_conf},
         {"quality_ph",          &quality_ph},
         {"class_ph",            &class_ph},
@@ -450,8 +449,8 @@ BathyDataFrame::Atl03Data::Atl03Data (const BathyDataFrame& dataframe, const Reg
     lat_ph              (dataframe.hdf03, FString("%s/%s", dataframe.beam.value.c_str(), "heights/lat_ph").c_str(),              0, region.first_photon,  region.num_photons),
     lon_ph              (dataframe.hdf03, FString("%s/%s", dataframe.beam.value.c_str(), "heights/lon_ph").c_str(),              0, region.first_photon,  region.num_photons),
     delta_time          (dataframe.hdf03, FString("%s/%s", dataframe.beam.value.c_str(), "heights/delta_time").c_str(),          0, region.first_photon,  region.num_photons),
-    bckgrd_delta_time   (dataframe.parms.findSeaSurface ? dataframe.hdf03 : NULL, FString("%s/%s", dataframe.beam.value.c_str(), "bckgrd_atlas/delta_time").c_str()),
-    bckgrd_rate         (dataframe.parms.findSeaSurface ? dataframe.hdf03 : NULL, FString("%s/%s", dataframe.beam.value.c_str(), "bckgrd_atlas/bckgrd_rate").c_str())
+    bckgrd_delta_time   (dataframe.hdf03, FString("%s/%s", dataframe.beam.value.c_str(), "bckgrd_atlas/delta_time").c_str()),
+    bckgrd_rate         (dataframe.hdf03, FString("%s/%s", dataframe.beam.value.c_str(), "bckgrd_atlas/bckgrd_rate").c_str())
 {
     sc_orient.join(dataframe.readTimeoutMs, true);
     velocity_sc.join(dataframe.readTimeoutMs, true);
@@ -474,8 +473,8 @@ BathyDataFrame::Atl03Data::Atl03Data (const BathyDataFrame& dataframe, const Reg
     lat_ph.join(dataframe.readTimeoutMs, true);
     lon_ph.join(dataframe.readTimeoutMs, true);
     delta_time.join(dataframe.readTimeoutMs, true);
-    if(dataframe.parms.findSeaSurface) bckgrd_delta_time.join(dataframe.readTimeoutMs, true);
-    if(dataframe.parms.findSeaSurface) bckgrd_rate.join(dataframe.readTimeoutMs, true);
+    bckgrd_delta_time.join(dataframe.readTimeoutMs, true);
+    bckgrd_rate.join(dataframe.readTimeoutMs, true);
 }
 
 /*----------------------------------------------------------------------------
@@ -674,6 +673,7 @@ void* BathyDataFrame::subsettingThread (void* parm)
 
                 /* Set Initial Processing Flags */
                 uint32_t processing_flags = BathyFields::FLAGS_CLEAR;
+                processing_flags |= static_cast<uint32_t>(yapc_score) << 24;
                 if(on_boundary) processing_flags |= BathyFields::ON_BOUNDARY;
                 if(atl03.solar_elevation[current_segment] < BathyFields::NIGHT_SOLAR_ELEVATION_THRESHOLD) processing_flags |= BathyFields::NIGHT_FLAG;
                 if(!atl09.valid) processing_flags |= BathyFields::INVALID_WIND_SPEED;
@@ -691,13 +691,12 @@ void* BathyDataFrame::subsettingThread (void* parm)
                 dataframe.y_atc.append(atl03.dist_ph_across[current_photon]);
                 dataframe.ellipse_h.append(atl03.h_ph[current_photon]); // later corrected by refraction correction
                 dataframe.ortho_h.append(atl03.h_ph[current_photon] - atl03.geoid[current_segment]); // later corrected by refraction correction
-                dataframe.yapc_score.append(yapc_score);
                 dataframe.max_signal_conf.append(atl03_cnf);
                 dataframe.quality_ph.append(quality_ph);
                 dataframe.processing_flags.append(processing_flags);
 
                 /* Add Additional Photon Data to DataFrame */
-                dataframe.background_rate.append(parms.findSeaSurface ? calculateBackground(current_segment, bckgrd_index, atl03) : 0.0);
+                dataframe.background_rate.append(calculateBackground(current_segment, bckgrd_index, atl03));
                 dataframe.geoid_corr_h.append(atl03.h_ph[current_photon] - atl03.geoid[current_segment]);
                 dataframe.wind_v.append(wind_v);
                 dataframe.ref_el.append(atl03.ref_elev[current_segment]);
@@ -750,9 +749,9 @@ void* BathyDataFrame::subsettingThread (void* parm)
 /*----------------------------------------------------------------------------
  * calculateBackground
  *----------------------------------------------------------------------------*/
-double BathyDataFrame::calculateBackground (int32_t current_segment, int32_t& bckgrd_index, const Atl03Data& atl03)
+float BathyDataFrame::calculateBackground (int32_t current_segment, int32_t& bckgrd_index, const Atl03Data& atl03)
 {
-    double background_rate = atl03.bckgrd_rate[atl03.bckgrd_rate.size - 1];
+    float background_rate = atl03.bckgrd_rate[atl03.bckgrd_rate.size - 1];
     while(bckgrd_index < atl03.bckgrd_rate.size)
     {
         const double curr_bckgrd_time = atl03.bckgrd_delta_time[bckgrd_index];

datasets/bathy/package/BathyDataFrame.h

@@ -93,18 +93,17 @@ class BathyDataFrame: public GeoDataFrame
         FieldColumn<double>         y_ph;               // the northing coordinate in meters of the photon for the given UTM zone
         FieldColumn<double>         x_atc;              // along track distance calculated from segment_dist_x and dist_ph_along
         FieldColumn<double>         y_atc;              // dist_ph_across
-        FieldColumn<double>         background_rate;    // PE per second
-        FieldColumn<float>          surface_h;          // orthometric height of sea surface at each photon location
         FieldColumn<double>         ortho_h {Field::Z_COLUMN}; // refraction corrected, geoid corrected height of photon
+        FieldColumn<float>          surface_h;          // orthometric height of sea surface at each photon location
         FieldColumn<float>          ellipse_h;          // height of photon with respect to reference ellipsoid
         FieldColumn<float>          sigma_thu;          // total horizontal uncertainty
         FieldColumn<float>          sigma_tvu;          // total vertical uncertainty
         FieldColumn<uint32_t>       processing_flags;   // bit mask of flags for capturing errors and warnings (top 8 bits reserved for classifiers)
-        FieldColumn<uint8_t>        yapc_score;         // atl03 density estimate (Yet Another Photon Classifier)
         FieldColumn<int8_t>         max_signal_conf;    // maximum value in the atl03 confidence table
         FieldColumn<int8_t>         quality_ph;         // atl03 quality flags
         FieldColumn<int8_t>         class_ph;           // photon classification
         FieldColumn<FieldArray<int8_t, BathyFields::NUM_CLASSIFIERS>> predictions; // photon classification from each of the classifiers
+        FieldColumn<float>          background_rate;    // PE per second
         FieldColumn<float>          geoid_corr_h;       // orthometric height without refraction correction (passed to classifiers)
         FieldColumn<float>          wind_v;             // wind speed (in meters/second)
         FieldColumn<float>          ref_el;             // reference elevation
@@ -226,7 +225,7 @@ class BathyDataFrame: public GeoDataFrame
                             ~BathyDataFrame             (void) override;
 
         static void*        subsettingThread            (void* parm);
-        static double       calculateBackground         (int32_t current_segment, int32_t& bckgrd_in, const Atl03Data& atl03);
+        static float        calculateBackground         (int32_t current_segment, int32_t& bckgrd_in, const Atl03Data& atl03);
 
         static int          luaIsValid                  (lua_State* L);
         static int          luaLength                   (lua_State* L);

datasets/bathy/package/BathyFields.cpp

@@ -117,7 +117,6 @@ BathyFields::BathyFields(lua_State* L, uint64_t key_space, const char* default_a
           {"ph_in_extent",        &phInExtent},
           {"generate_ndwi",       &generateNdwi},
           {"use_bathy_mask",      &useBathyMask},
-          {"find_sea_surface",    &findSeaSurface},
           {"classifiers",         &classifiers},
           {"spots",               &spots},
           {"surface",             &surface},