Merge pull request #433 from SlideRuleEarth/new_parquet_sampler

Implemented batch processing for the indexed raster class.
SlideRuleEarth · Sep 23, 2024 · da9e4ae · da9e4ae
2 parents 16aca7b + 6442272
commit da9e4ae
Show file tree

Hide file tree

Showing 33 changed files with 2,406 additions and 817 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -52,6 +52,7 @@ if(CMAKE_BUILD_TYPE MATCHES "Debug")
         "-misc-non-private-member-variables-in-classes,"
         "-misc-include-cleaner,"
         "-misc-use-anonymous-namespace,"
+        "-misc-no-recursion,"
 
         "hicpp-*,"
         "-hicpp-special-member-functions,"
@@ -139,6 +140,8 @@ if(CMAKE_BUILD_TYPE MATCHES "Debug")
         "--suppress=constParameterReference:*/BathyRefractionCorrector.cpp" # in run() param extent cannot be const
         "--suppress=constParameterPointer:*/CcsdsPayloadDispatch.cpp"   # Not trivial to fix, would have to change DispachObject class as well.
         "--suppress=knownConditionTrueFalse"                            # -1 < 0 etc
+        "--suppress=constVariableReference:*/RasterObject.cpp"          # List [] const issue
+        "--suppress=constVariableReference:*/GeoIndexedRaster.cpp"      # List [] const issue
     )
 
 endif()

diff --git a/datasets/gebco/package/GebcoBathyRaster.cpp b/datasets/gebco/package/GebcoBathyRaster.cpp
@@ -70,9 +70,10 @@ GebcoBathyRaster::~GebcoBathyRaster(void) = default;
 /*----------------------------------------------------------------------------
  * getIndexFile
  *----------------------------------------------------------------------------*/
-void GebcoBathyRaster::getIndexFile(const OGRGeometry* geo, std::string& file)
+void GebcoBathyRaster::getIndexFile(const OGRGeometry* geo, std::string& file, const std::vector<point_info_t>* points)
 {
     static_cast<void>(geo);
+    static_cast<void>(points);
     file = filePath + "/" + indexFile;
     mlog(DEBUG, "Using %s", file.c_str());
 }
@@ -83,17 +84,17 @@ void GebcoBathyRaster::getIndexFile(const OGRGeometry* geo, std::string& file)
  *----------------------------------------------------------------------------*/
 bool GebcoBathyRaster::findRasters(finder_t* finder)
 {
-    const OGRGeometry* geo    = finder->geo;
-    const uint32_t start_indx = finder->range.start_indx;
-    const uint32_t end_indx   = finder->range.end_indx;
+    const std::vector<OGRFeature*>* flist = finder->featuresList;
+    const OGRGeometry* geo = finder->geo;
+    const uint32_t start   = 0;
+    const uint32_t end     = flist->size();
 
     try
     {
-        for(uint32_t i = start_indx; i < end_indx; i++)
+        for(uint32_t i = start; i < end; i++)
         {
-            OGRFeature* feature = featuresList[i];
+            OGRFeature* feature = flist->at(i);
             OGRGeometry *rastergeo = feature->GetGeometryRef();
-            CHECKPTR(geo);
 
             if (!rastergeo->Intersects(geo)) continue;
 

diff --git a/datasets/gebco/package/GebcoBathyRaster.h b/datasets/gebco/package/GebcoBathyRaster.h
@@ -71,7 +71,7 @@ class GebcoBathyRaster: public GeoIndexedRaster
                 GebcoBathyRaster (lua_State* L, GeoParms* _parms);
                ~GebcoBathyRaster (void) override;
 
-        void    getIndexFile     (const OGRGeometry* geo, std::string& file) final;
+        void    getIndexFile     (const OGRGeometry* geo, std::string& file, const std::vector<point_info_t>* points) final;
         bool    findRasters      (finder_t* finder) final;
 
         /*--------------------------------------------------------------------

diff --git a/datasets/icesat2/package/MeritRaster.cpp b/datasets/icesat2/package/MeritRaster.cpp
@@ -117,6 +117,8 @@ uint32_t MeritRaster::getSamples (const MathLib::point_3d_t& point, int64_t gps,
     (void)param;
     (void)gps;
 
+    lockSampling();
+
     /* Determine Upper Left Coordinates */
     int left_lon = ((int)floor(point.x / 5.0)) * 5;
     int upper_lat = ((int)ceil(point.y / 5.0)) * 5;
@@ -203,6 +205,8 @@ uint32_t MeritRaster::getSamples (const MathLib::point_3d_t& point, int64_t gps,
         mlog(ERROR, "Failed to sample dataset: %s", dataset.c_str());
     }
 
+    unlockSampling();
+
     return SS_NO_ERRORS;
 }