Skip to content

Commit

Permalink
run black for code formating
Browse files Browse the repository at this point in the history
  • Loading branch information
simonbesnard1 committed Feb 24, 2025
1 parent 680c0c8 commit b0c18ed
Show file tree
Hide file tree
Showing 6 changed files with 50 additions and 28 deletions.
28 changes: 19 additions & 9 deletions gedidb/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,20 @@
from gedidb.downloader import authentication, cmr_query, data_downloader
from gedidb.downloader.authentication import EarthDataAuthenticator
from gedidb.downloader.cmr_query import CMRQuery
from gedidb.downloader.data_downloader import (CMRDataDownloader,
GEDIDownloader,
H5FileDownloader)
from gedidb.granule import (Granule, granule_name, granule_parser, l2a_granule,
l2b_granule, l4a_granule, l4c_granule)
from gedidb.downloader.data_downloader import (
CMRDataDownloader,
GEDIDownloader,
H5FileDownloader,
)
from gedidb.granule import (
Granule,
granule_name,
granule_parser,
l2a_granule,
l2b_granule,
l4a_granule,
l4c_granule,
)
from gedidb.granule.Granule import granule_handler
from gedidb.granule.granule_name import GediNameMetadata
from gedidb.granule.granule_parser import GranuleParser
Expand All @@ -44,10 +53,11 @@
from gedidb.granule.l4c_granule import L4CGranule
from gedidb.providers import tiledb_provider
from gedidb.providers.tiledb_provider import TileDBProvider
from gedidb.utils import (constants, geo_processing, print_versions,
tiledb_consolidation)
from gedidb.utils.tiledb_consolidation import (SpatialConsolidationPlan,
SpatialConsolidationPlanner)
from gedidb.utils import constants, geo_processing, print_versions, tiledb_consolidation
from gedidb.utils.tiledb_consolidation import (
SpatialConsolidationPlan,
SpatialConsolidationPlanner,
)

__all__ = [
"gedidatabase",
Expand Down
10 changes: 6 additions & 4 deletions gedidb/core/gedidatabase.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@
from dask.distributed import Client
from retry import retry

from gedidb.utils.geo_processing import (_datetime_to_timestamp_days,
convert_to_days_since_epoch)
from gedidb.utils.geo_processing import (
_datetime_to_timestamp_days,
convert_to_days_since_epoch,
)
from gedidb.utils.tiledb_consolidation import SpatialConsolidationPlanner

# Configure the logger
Expand Down Expand Up @@ -217,14 +219,14 @@ def consolidate_fragments(
logger.info("Executing consolidation...")
self._execute_consolidation(cons_plan, parallel_engine)
logger.info("Consolidation execution completed.")

logger.info("Consolidating array metadata...")
self._consolidate_and_vacuum("array_meta")
logger.info("Consolidating fragment metadata...")
self._consolidate_and_vacuum("fragment_meta")
logger.info("Consolidating commit logs...")
self._consolidate_and_vacuum("commits")

logger.info(f"Consolidation complete for {self.array_uri}")

except tiledb.TileDBError as e:
Expand Down
6 changes: 2 additions & 4 deletions gedidb/core/gediprocessor.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,9 @@
from gedidb.core.gedidatabase import GEDIDatabase
from gedidb.core.gedigranule import GEDIGranule
from gedidb.downloader.authentication import EarthDataAuthenticator
from gedidb.downloader.data_downloader import (CMRDataDownloader,
H5FileDownloader)
from gedidb.downloader.data_downloader import CMRDataDownloader, H5FileDownloader
from gedidb.utils.constants import GediProduct
from gedidb.utils.geo_processing import (_temporal_tiling,
check_and_format_shape)
from gedidb.utils.geo_processing import _temporal_tiling, check_and_format_shape

# Configure logging
logging.basicConfig(
Expand Down
8 changes: 5 additions & 3 deletions gedidb/core/gediprovider.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@
from scipy.spatial import cKDTree

from gedidb.providers.tiledb_provider import TileDBProvider
from gedidb.utils.geo_processing import (_datetime_to_timestamp_days,
_timestamp_to_datetime,
check_and_format_shape)
from gedidb.utils.geo_processing import (
_datetime_to_timestamp_days,
_timestamp_to_datetime,
check_and_format_shape,
)

# Configure the logger
logging.basicConfig(
Expand Down
23 changes: 17 additions & 6 deletions gedidb/downloader/data_downloader.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,14 @@

import geopandas as gpd
import requests
from requests.exceptions import (ChunkedEncodingError, ConnectionError,
HTTPError, ReadTimeout, RequestException,
Timeout)
from requests.exceptions import (
ChunkedEncodingError,
ConnectionError,
HTTPError,
ReadTimeout,
RequestException,
Timeout,
)
from retry import retry
from urllib3.exceptions import NewConnectionError
import h5py
Expand Down Expand Up @@ -219,7 +224,9 @@ def download(
if self._is_hdf5_valid(final_path):
return granule_key, (product.value, str(final_path))
else:
logger.warning(f"Corrupt HDF5 file detected: {final_path}. Deleting and retrying.")
logger.warning(
f"Corrupt HDF5 file detected: {final_path}. Deleting and retrying."
)
final_path.unlink()

# Get the size of partially downloaded file
Expand All @@ -243,7 +250,9 @@ def download(
if self._is_hdf5_valid(final_path):
return granule_key, (product.value, str(final_path))
else:
logger.warning(f"Downloaded file {final_path} is corrupt. Deleting and retrying.")
logger.warning(
f"Downloaded file {final_path} is corrupt. Deleting and retrying."
)
final_path.unlink()
raise ValueError("Invalid HDF5 file after download.")
headers["Range"] = f"bytes={downloaded_size}-"
Expand Down Expand Up @@ -281,7 +290,9 @@ def download(

# Validate the HDF5 file
if not self._is_hdf5_valid(final_path):
logger.warning(f"Downloaded file {final_path} is corrupt. Deleting and retrying.")
logger.warning(
f"Downloaded file {final_path} is corrupt. Deleting and retrying."
)
final_path.unlink()
raise ValueError("Invalid HDF5 file after download.")

Expand Down
3 changes: 1 addition & 2 deletions gedidb/granule/Granule.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,7 @@
import pandas as pd

from gedidb.beam.Beam import beam_handler
from gedidb.granule.granule_name import (GediNameMetadata,
parse_granule_filename)
from gedidb.granule.granule_name import GediNameMetadata, parse_granule_filename


class granule_handler(h5py.File):
Expand Down

0 comments on commit b0c18ed

Please sign in to comment.