Add transfer configs (#108)

NASA-IMPACT · Mar 18, 2024 · 867148e · 867148e
1 parent 1de7f4d
commit 867148e
Show file tree

Hide file tree

Showing 157 changed files with 3,788 additions and 147 deletions.
diff --git a/ingestion-data/discovery-items/OMSO2PCA-cog.json b/ingestion-data/discovery-items/OMSO2PCA-cog.json
@@ -2,7 +2,7 @@
     "collection": "OMSO2PCA-COG",
     "bucket": "veda-data-store-staging",
     "prefix": "OMSO2PCA-COG/",
-    "filename_regex":"OMSO2PCA_LUT_SCD",
+    "filename_regex":"^(.*)OMSO2PCA_LUT_SCD",
     "discovery": "s3",
     "upload": false,
     "datetime_range": "year"

diff --git a/ingestion-data/discovery-items/barc-thomasfire.json b/ingestion-data/discovery-items/barc-thomasfire.json
@@ -2,7 +2,7 @@
     "collection": "barc-thomasfire",
     "bucket": "veda-data-store-staging",
     "prefix": "EIS/COG/Fire-Hydro/",
-    "filename_regex": "thomas_fire_barc_201712",
+    "filename_regex": "^(.*)thomas_fire_barc_201712",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/campfire-albedo-wsa-diff.json b/ingestion-data/discovery-items/campfire-albedo-wsa-diff.json
@@ -2,7 +2,7 @@
     "collection": "campfire-albedo-wsa-diff",
     "bucket": "veda-data-store-staging",
     "prefix": "campfire-data/",
-    "filename_regex": "CampFire_albedo_WSA_2015_2022_diff",
+    "filename_regex": "^(.*)CampFire_albedo_WSA_2015_2022_diff",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/campfire-lst-day-diff.json b/ingestion-data/discovery-items/campfire-lst-day-diff.json
@@ -2,7 +2,7 @@
     "collection": "campfire-lst-day-diff",
     "bucket": "veda-data-store-staging",
     "prefix": "campfire-data/",
-    "filename_regex": "CampFire_LST_Day_2015_2022_diff",
+    "filename_regex": "^(.*)CampFire_LST_Day_2015_2022_diff",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/campfire-lst-night-diff.json b/ingestion-data/discovery-items/campfire-lst-night-diff.json
@@ -2,7 +2,7 @@
     "collection": "campfire-lst-night-diff",
     "bucket": "veda-data-store-staging",
     "prefix": "campfire-data/",
-    "filename_regex": "CampFire_LST_Night_2015_2022_diff",
+    "filename_regex": "^(.*)CampFire_LST_Night_2015_2022_diff",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/campfire-ndvi-diff.json b/ingestion-data/discovery-items/campfire-ndvi-diff.json
@@ -2,7 +2,7 @@
     "collection": "campfire-ndvi-diff",
     "bucket": "veda-data-store-staging",
     "prefix": "campfire-data/",
-    "filename_regex": "CampFire_NDVI_2015_2022_diff",
+    "filename_regex": "^(.*)CampFire_NDVI_2015_2022_diff",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/campfire-nlcd.json b/ingestion-data/discovery-items/campfire-nlcd.json
@@ -2,7 +2,7 @@
     "collection": "campfire-nlcd",
     "bucket": "veda-data-store-staging",
     "prefix": "campfire-data/",
-    "filename_regex": "CampFire_NLCD_2019",
+    "filename_regex": "^(.*)CampFire_NLCD_2019",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/ceos-co2-flux-budgets-mean.json b/ingestion-data/discovery-items/ceos-co2-flux-budgets-mean.json
diff --git a/ingestion-data/discovery-items/ceos-co2-flux-budgets.json b/ingestion-data/discovery-items/ceos-co2-flux-budgets.json
diff --git a/ingestion-data/discovery-items/conus-reach.json b/ingestion-data/discovery-items/conus-reach.json
@@ -3,6 +3,6 @@
     "bucket": "veda-data-store-staging",
     "prefix":"EIS/COG/Water-Quality/",
     "discovery": "s3",
-    "filename_regex": "CONUS_Reach_2020",
+    "filename_regex": "^(.*)CONUS_Reach_2020",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/damage_probability_2022-10-03.json b/ingestion-data/discovery-items/damage_probability_2022-10-03.json
@@ -3,6 +3,6 @@
     "bucket": "veda-data-store-staging",
     "prefix":"damage-probability/",
     "discovery": "s3",
-    "filename_regex":"damage_probability_cog",
+    "filename_regex":"^(.*)damage_probability_cog",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/disalexi-etsuppression.json b/ingestion-data/discovery-items/disalexi-etsuppression.json
@@ -2,7 +2,7 @@
     "collection": "disalexi-etsuppression",
     "bucket": "veda-data-store-staging",
     "prefix":"EIS/COG/Fire-Hydro/",
-    "filename_regex": "disalexi_etsuppression",
+    "filename_regex": "^(.*)disalexi_etsuppression",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/disturbance-probability-percentile.json b/ingestion-data/discovery-items/disturbance-probability-percentile.json
@@ -3,6 +3,6 @@
     "bucket": "veda-data-store-staging",
     "prefix":"disturbance-probability-percentile",
     "discovery": "s3",
-    "filename_regex":"spec_prob_mosaic",
+    "filename_regex":"^(.*)spec_prob_mosaic",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/entropy_difference_2022-09-05_2022-09-30.json b/ingestion-data/discovery-items/entropy_difference_2022-09-05_2022-09-30.json
@@ -2,7 +2,7 @@
     "collection": "entropy_difference_2022-09-05_2022-09-30",
     "bucket": "veda-data-store-staging",
     "prefix":"hls-entropy",
-    "filename_regex":"entropy_difference",
+    "filename_regex":"^(.*)entropy_difference",
     "discovery": "s3",
     "upload": false,
     "start_datetime": "2022-09-05T00:00:00Z",

diff --git a/ingestion-data/discovery-items/fldas-soil-moisture-anomalies.json b/ingestion-data/discovery-items/fldas-soil-moisture-anomalies.json
@@ -3,6 +3,6 @@
     "bucket": "veda-data-store-staging",
     "prefix":"fldas-soil-moisture-anomalies",
     "discovery": "s3",
-    "filename_regex":"FLDAS_NOAH01_SoilMoi00_10cm_tavg",
+    "filename_regex":"^(.*)FLDAS_NOAH01_SoilMoi00_10cm_tavg",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/frp-max-thomasfire.json b/ingestion-data/discovery-items/frp-max-thomasfire.json
@@ -2,7 +2,7 @@
     "collection": "frp-max-thomasfire",
     "bucket": "veda-data-store-staging",
     "prefix":"EIS/COG/Fire-Hydro/",
-    "filename_regex":"thomas_fire_frpmax",
+    "filename_regex":"^(.*)thomas_fire_frpmax",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/hls-bais2-v2.json b/ingestion-data/discovery-items/hls-bais2-v2.json
@@ -2,7 +2,7 @@
     "collection": "hls-bais2-v2",
     "bucket": "veda-data-store-staging",
     "prefix":"maui-fire",
-    "filename_regex":"Lahaina_BAIS2",
+    "filename_regex":"^(.*)Lahaina_BAIS2",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/hls-bais2.json b/ingestion-data/discovery-items/hls-bais2.json
@@ -2,7 +2,7 @@
     "collection": "hls-bais2",
     "bucket": "veda-data-store-staging",
     "prefix":"maui-fire",
-    "filename_regex":"Lahaina_BAIS2",
+    "filename_regex":"^(.*)Lahaina_BAIS2",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/hls-ndvi.json b/ingestion-data/discovery-items/hls-ndvi.json
@@ -2,7 +2,7 @@
     "collection": "hls-ndvi",
     "bucket": "veda-data-store-staging",
     "prefix":"hls-ndvi",
-    "filename_regex":"ndvi_cog_2022-09-05",
+    "filename_regex":"^(.*)ndvi_cog_2022-09-05",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/hls-swir-falsecolor-composite.json b/ingestion-data/discovery-items/hls-swir-falsecolor-composite.json
@@ -2,7 +2,7 @@
     "collection": "hls-swir-falsecolor-composite",
     "bucket": "veda-data-store-staging",
     "prefix":"maui-fire",
-    "filename_regex":"SWIR_falsecolor_cog",
+    "filename_regex":"^(.*)SWIR_falsecolor_cog",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/houston-aod-diff.json b/ingestion-data/discovery-items/houston-aod-diff.json
@@ -2,7 +2,7 @@
     "collection": "houston-aod-diff",
     "bucket": "veda-data-store-staging",
     "prefix":"houston-aod/",
-    "filename_regex":"HOUS_AOD_difference_2000_2019",
+    "filename_regex":"^(.*)HOUS_AOD_difference_2000_2019",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/houston-landcover.json b/ingestion-data/discovery-items/houston-landcover.json
@@ -2,7 +2,7 @@
     "collection": "houston-landcover",
     "bucket": "veda-data-store-staging",
     "prefix":"houston-landcover/",
-    "filename_regex":"landcover_Z_f",
+    "filename_regex":"^(.*)landcover_Z_f",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/houston-lst-day.json b/ingestion-data/discovery-items/houston-lst-day.json
@@ -2,7 +2,7 @@
     "collection": "houston-lst-day",
     "bucket": "veda-data-store-staging",
     "prefix":"houston-lst-day/",
-    "filename_regex":"LST_Day",
+    "filename_regex":"^(.*)LST_Day",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/houston-lst-diff.json b/ingestion-data/discovery-items/houston-lst-diff.json
@@ -2,7 +2,7 @@
     "collection": "houston-lst-diff",
     "bucket": "climatedashboard-data",
     "prefix":"houston/",
-    "filename_regex":"houston-lst-diff_2000_2019",
+    "filename_regex":"^(.*)houston-lst-diff_2000_2019",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/houston-lst-night.json b/ingestion-data/discovery-items/houston-lst-night.json
@@ -2,7 +2,7 @@
     "collection": "houston-lst-night",
     "bucket": "veda-data-store-staging",
     "prefix":"houston-lst-night/",
-    "filename_regex":"LST_Night",
+    "filename_regex":"^(.*)LST_Night",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/houston-ndvi.json b/ingestion-data/discovery-items/houston-ndvi.json
@@ -2,7 +2,7 @@
     "collection": "houston-ndvi",
     "bucket": "veda-data-store-staging",
     "prefix": "houston-ndvi/",
-    "filename_regex":"NDVI",
+    "filename_regex":"^(.*)NDVI",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/houston-urbanization.json b/ingestion-data/discovery-items/houston-urbanization.json
@@ -2,7 +2,7 @@
     "collection": "houston-urbanization",
     "bucket": "veda-data-store-staging",
     "prefix": "urbanization/", 
-    "filename_regex":"HOUS_DFAL_urbanization",
+    "filename_regex":"^(.*)HOUS_DFAL_urbanization",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/landsat-nighttime-thermal.json b/ingestion-data/discovery-items/landsat-nighttime-thermal.json
@@ -2,7 +2,7 @@
     "collection": "landsat-nighttime-thermal",
     "bucket": "veda-data-store-staging",
     "prefix": "maui-fire/",
-    "filename_regex":"Lahaina_Landsat8_Thermal",
+    "filename_regex":"^(.*)Lahaina_Landsat8_Thermal",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/lis-etsuppression.json b/ingestion-data/discovery-items/lis-etsuppression.json
@@ -2,7 +2,7 @@
     "collection": "lis-etsuppression",
     "bucket": "veda-data-store-staging",
     "prefix":"EIS/COG/Fire-Hydro/",
-    "filename_regex":"lis_etsuppression_2020",
+    "filename_regex":"^(.*)lis_etsuppression_2020",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/lis-global-da-evap.json b/ingestion-data/discovery-items/lis-global-da-evap.json
@@ -2,7 +2,7 @@
     "collection": "lis-global-da-evap",
     "bucket": "veda-data-store-staging",
     "prefix":"EIS/COG/LIS_GLOBAL_DA/Evap/",
-    "filename_regex":"LIS_Evap",
+    "filename_regex":"^(.*)LIS_Evap",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/lis-global-da-gpp-trend.json b/ingestion-data/discovery-items/lis-global-da-gpp-trend.json
@@ -2,7 +2,7 @@
     "collection": "lis-global-da-gpp-trend",
     "bucket": "veda-data-store-staging",
     "prefix":"EIS/COG/LIS_GLOBAL_DA/DA_Trends/",
-    "filename_regex":"",
+    "filename_regex":"^(.*)DAGPP_STL_based_trend",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/lis-global-da-gpp.json b/ingestion-data/discovery-items/lis-global-da-gpp.json
@@ -2,7 +2,7 @@
     "collection": "lis-global-da-gpp",
     "bucket": "veda-data-store-staging",
     "prefix": "EIS/COG/LIS_GLOBAL_DA/GPP/",
-    "filename_regex":"LIS_GPP",
+    "filename_regex":"^(.*)LIS_GPP",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/lis-global-da-gws.json b/ingestion-data/discovery-items/lis-global-da-gws.json
@@ -2,7 +2,7 @@
     "collection": "lis-global-da-gws",
     "bucket": "veda-data-store-staging",
     "prefix": "EIS/COG/LIS_GLOBAL_DA/GWS/",
-    "filename_regex":"LIS_GWS",
+    "filename_regex":"^(.*)LIS_GWS",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/lis-global-da-qs.json b/ingestion-data/discovery-items/lis-global-da-qs.json
@@ -2,7 +2,7 @@
     "collection": "lis-global-da-qs",
     "bucket": "veda-data-store-staging",
     "prefix": "EIS/COG/LIS_GLOBAL_DA/Qs/",
-    "filename_regex":"LIS_Qs",
+    "filename_regex":"^(.*)LIS_Qs",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/lis-global-da-qsb.json b/ingestion-data/discovery-items/lis-global-da-qsb.json
@@ -2,7 +2,7 @@
     "collection": "lis-global-da-qsb",
     "bucket": "veda-data-store-staging",
     "prefix": "EIS/COG/LIS_GLOBAL_DA/Qsb/",
-    "filename_regex":"LIS_Qsb",
+    "filename_regex":"^(.*)LIS_Qsb",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/lis-global-da-streamflow.json b/ingestion-data/discovery-items/lis-global-da-streamflow.json
@@ -2,7 +2,7 @@
     "collection": "lis-global-da-streamflow",
     "bucket": "veda-data-store-staging",
     "prefix": "EIS/COG/LIS_GLOBAL_DA/Streamflow/",
-    "filename_regex":"LIS_Streamflow",
+    "filename_regex":"^(.*)LIS_Streamflow",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/lis-global-da-swe.json b/ingestion-data/discovery-items/lis-global-da-swe.json
@@ -1,8 +1,8 @@
 {
     "collection": "lis-global-da-swe",
     "bucket": "veda-data-store-staging",
-    "prefix": "lis-global-da-swe",
-    "filename_regex":"LIS_SWE",
+    "prefix": "EIS/COG/LIS_GLOBAL_DA/SWE/",
+    "filename_regex":"^(.*)LIS_SWE",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/lis-global-da-totalprecip.json b/ingestion-data/discovery-items/lis-global-da-totalprecip.json
@@ -2,7 +2,7 @@
     "collection": "lis-global-da-totalprecip",
     "bucket": "veda-data-store-staging",
     "prefix": "EIS/COG/LIS_GLOBAL_DA/TotalPrecip/",
-    "filename_regex":"LIS_TotalPrecip",
+    "filename_regex":"^(.*)LIS_TotalPrecip",
     "discovery": "s3",
     "upload": false
 }
diff --git a/ingestion-data/discovery-items/lis-global-da-tws-trend.json b/ingestion-data/discovery-items/lis-global-da-tws-trend.json
@@ -3,23 +3,23 @@
         "collection": "lis-global-da-tws-trend",
         "bucket": "veda-data-store-staging",
         "prefix": "EIS/COG/LIS_GLOBAL_DA/DA_Trends/",
-        "filename_regex":"DATWS_STL_based_trend_v3_2003_2021",
+        "filename_regex":"^(.*)DATWS_STL_based_trend_v3_2003_2021",
         "discovery": "s3",
         "upload": false
     },
     {
         "collection": "lis-global-da-tws-trend",
         "bucket": "veda-data-store-staging",
         "prefix": "EIS/COG/LIS_GLOBAL_DA/DA_Trends/",
-        "filename_regex":"DATWS_STL_based_trendv2_2003_2021",
+        "filename_regex":"^(.*)DATWS_STL_based_trendv2_2003_2021",
         "discovery": "s3",
         "upload": false
     },
     {
         "collection": "lis-global-da-tws-trend",
         "bucket": "veda-data-store-staging",
         "prefix": "EIS/COG/LIS_GLOBAL_DA/DA_Trends/",
-        "filename_regex":"DATWS_STL_based_trend.cog.tif",
+        "filename_regex":"^(.*)DATWS_STL_based_trend.cog.tif",
         "discovery": "s3",
         "upload": false
     }

diff --git a/ingestion-data/discovery-items/lis-global-da-tws.json b/ingestion-data/discovery-items/lis-global-da-tws.json
@@ -2,7 +2,7 @@
     "collection": "lis-global-da-tws",
     "bucket": "veda-data-store-staging",
     "prefix": "EIS/COG/LIS_GLOBAL_DA/TWS/",
-    "filename_regex":"LIS_TWS",
+    "filename_regex":"^(.*)LIS_TWS",
     "discovery": "s3",
     "upload": false
 }