Merge pull request #139 from ecmwf/develop

prepare 0.5.11

pyproject.toml

@@ -49,8 +49,8 @@ dynamic = [
   "version",
 ]
 dependencies = [
-  "anemoi-transform>=0.0.5",
-  "anemoi-utils[provenance]>=0.4.2",
+  "anemoi-transform>=0.1",
+  "anemoi-utils[provenance]>=0.4.9",
   "cfunits",
   "numpy",
   "pyyaml",

src/anemoi/datasets/create/__init__.py

@@ -19,7 +19,6 @@
 import cftime
 import numpy as np
 import tqdm
-from anemoi.utils.config import DotDict as DotDict
 from anemoi.utils.dates import as_datetime
 from anemoi.utils.dates import frequency_to_string
 from anemoi.utils.dates import frequency_to_timedelta
@@ -327,7 +326,7 @@ def create_elements(self, config):
         self.output = build_output(config.output, parent=self)
 
         self.input = build_input_(main_config=config, output_config=self.output)
-        LOG.info(self.input)
+        LOG.info("%s", self.input)
 
 
 def build_input_(main_config, output_config):

src/anemoi/datasets/create/functions/filters/rename.py

@@ -44,7 +44,6 @@ def __getattr__(self, name):
 
     def __repr__(self) -> str:
         return repr(self.field)
-        return f"{self.field} -> {self.what} -> {self.renaming}"
 
 
 class RenamedFieldFormat:

src/anemoi/datasets/create/functions/sources/grib.py

@@ -9,11 +9,14 @@
 
 
 import glob
+import logging
 
 from earthkit.data import from_source
 from earthkit.data.indexing.fieldlist import FieldArray
 from earthkit.data.utils.patterns import Pattern
 
+LOG = logging.getLogger(__name__)
+
 
 def _load(context, name, record):
     ds = None
@@ -141,4 +144,7 @@ def execute(context, dates, path, latitudes=None, longitudes=None, *args, **kwar
     if geography is not None:
         ds = FieldArray([AddGrid(_, geography) for _ in ds])
 
+    if len(ds) == 0:
+        LOG.warning(f"No fields found for {dates} in {given_paths} (kwargs={kwargs})")
+
     return ds

src/anemoi/datasets/create/functions/sources/xarray/__init__.py

@@ -12,7 +12,6 @@
 from earthkit.data.core.fieldlist import MultiFieldList
 
 from anemoi.datasets.data.stores import name_to_zarr_store
-from anemoi.datasets.utils.fields import NewMetadataField as NewMetadataField
 
 from .. import iterate_patterns
 from .fieldlist import XarrayFieldList

src/anemoi/datasets/create/functions/sources/xarray/field.py

@@ -10,6 +10,7 @@
 
 import datetime
 import logging
+from functools import cached_property
 
 from earthkit.data.core.fieldlist import Field
 from earthkit.data.core.fieldlist import math
@@ -46,8 +47,6 @@ def __init__(self, owner, selection):
             This is actually a nD object, but the first dimensions are always 1.
             The other two dimensions are latitude and longitude.
         """
-        super().__init__(owner.array_backend)
-
         self.owner = owner
         self.selection = selection
 
@@ -86,7 +85,8 @@ def to_numpy(self, flatten=False, dtype=None, index=None):
 
         return values  # .reshape(self.shape)
 
-    def _make_metadata(self):
+    @cached_property
+    def _metadata(self):
         return XArrayMetadata(self)
 
     def grid_points(self):

src/anemoi/datasets/create/functions/sources/xarray/fieldlist.py

@@ -14,10 +14,8 @@
 import yaml
 from earthkit.data.core.fieldlist import FieldList
 
-from .coordinates import is_scalar as is_scalar
 from .field import EmptyFieldList
 from .flavour import CoordinateGuesser
-from .metadata import XArrayMetadata as XArrayMetadata
 from .time import Time
 from .variable import FilteredVariable
 from .variable import Variable

src/anemoi/datasets/create/functions/sources/xarray/variable.py

@@ -13,7 +13,6 @@
 from functools import cached_property
 
 import numpy as np
-from earthkit.data.utils.array import ensure_backend
 
 from .field import XArrayField
 
@@ -30,7 +29,6 @@ def __init__(
         grid,
         time,
         metadata,
-        array_backend=None,
     ):
         self.ds = ds
         self.variable = variable
@@ -48,7 +46,6 @@ def __init__(
         self.by_name = {c.variable.name: c for c in coordinates}
 
         self.length = math.prod(self.shape)
-        self.array_backend = ensure_backend(array_backend)
 
     @property
     def name(self):

src/anemoi/datasets/create/input/action.py

@@ -10,14 +10,8 @@
 import logging
 from copy import deepcopy
 
-from anemoi.utils.dates import as_datetime as as_datetime
-from anemoi.utils.dates import frequency_to_timedelta as frequency_to_timedelta
 from earthkit.data.core.order import build_remapping
 
-from anemoi.datasets.dates import DatesProvider as DatesProvider
-from anemoi.datasets.fields import FieldArray as FieldArray
-from anemoi.datasets.fields import NewValidDateTimeField as NewValidDateTimeField
-
 from .context import Context
 from .misc import is_function
 

src/anemoi/datasets/create/input/context.py

@@ -10,14 +10,8 @@
 import logging
 import textwrap
 
-from anemoi.utils.dates import as_datetime as as_datetime
-from anemoi.utils.dates import frequency_to_timedelta as frequency_to_timedelta
 from anemoi.utils.humanize import plural
 
-from anemoi.datasets.dates import DatesProvider as DatesProvider
-from anemoi.datasets.fields import FieldArray as FieldArray
-from anemoi.datasets.fields import NewValidDateTimeField as NewValidDateTimeField
-
 from .trace import step
 from .trace import trace
 

src/anemoi/datasets/create/input/data_sources.py

@@ -10,13 +10,6 @@
 import logging
 from functools import cached_property
 
-from anemoi.utils.dates import as_datetime as as_datetime
-from anemoi.utils.dates import frequency_to_timedelta as frequency_to_timedelta
-
-from anemoi.datasets.dates import DatesProvider as DatesProvider
-from anemoi.datasets.fields import FieldArray as FieldArray
-from anemoi.datasets.fields import NewValidDateTimeField as NewValidDateTimeField
-
 from .action import Action
 from .action import action_factory
 from .misc import _tidy

src/anemoi/datasets/create/input/empty.py

@@ -10,13 +10,6 @@
 import logging
 from functools import cached_property
 
-from anemoi.utils.dates import as_datetime as as_datetime
-from anemoi.utils.dates import frequency_to_timedelta as frequency_to_timedelta
-
-from anemoi.datasets.dates import DatesProvider as DatesProvider
-from anemoi.datasets.fields import FieldArray as FieldArray
-from anemoi.datasets.fields import NewValidDateTimeField as NewValidDateTimeField
-
 from .misc import assert_fieldlist
 from .result import Result
 from .trace import trace_datasource

src/anemoi/datasets/create/input/filter.py

@@ -10,13 +10,6 @@
 import logging
 from functools import cached_property
 
-from anemoi.utils.dates import as_datetime as as_datetime
-from anemoi.utils.dates import frequency_to_timedelta as frequency_to_timedelta
-
-from anemoi.datasets.dates import DatesProvider as DatesProvider
-from anemoi.datasets.fields import FieldArray as FieldArray
-from anemoi.datasets.fields import NewValidDateTimeField as NewValidDateTimeField
-
 from ..functions import import_function
 from .function import FunctionContext
 from .misc import _tidy

src/anemoi/datasets/create/input/function.py

@@ -10,13 +10,6 @@
 import logging
 from functools import cached_property
 
-from anemoi.utils.dates import as_datetime as as_datetime
-from anemoi.utils.dates import frequency_to_timedelta as frequency_to_timedelta
-
-from anemoi.datasets.dates import DatesProvider as DatesProvider
-from anemoi.datasets.fields import FieldArray as FieldArray
-from anemoi.datasets.fields import NewValidDateTimeField as NewValidDateTimeField
-
 from ..functions import import_function
 from .action import Action
 from .misc import _tidy

src/anemoi/datasets/create/input/misc.py

@@ -10,15 +10,9 @@
 import logging
 from functools import wraps
 
-from anemoi.utils.dates import as_datetime as as_datetime
-from anemoi.utils.dates import frequency_to_timedelta as frequency_to_timedelta
 from earthkit.data.core.fieldlist import MultiFieldList
 from earthkit.data.indexing.fieldlist import FieldList
 
-from anemoi.datasets.dates import DatesProvider as DatesProvider
-from anemoi.datasets.fields import FieldArray as FieldArray
-from anemoi.datasets.fields import NewValidDateTimeField as NewValidDateTimeField
-
 from ..functions import import_function
 
 LOG = logging.getLogger(__name__)

src/anemoi/datasets/create/input/pipe.py

@@ -7,6 +7,7 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
+import json
 import logging
 
 from .action import Action
@@ -20,7 +21,11 @@
 class PipeAction(Action):
     def __init__(self, context, action_path, *configs):
         super().__init__(context, action_path, *configs)
-        assert len(configs) > 1, configs
+        if len(configs) <= 1:
+            raise ValueError(
+                f"PipeAction requires at least two actions, got {len(configs)}\n{json.dumps(configs, indent=2)}"
+            )
+
         current = action_factory(configs[0], context, action_path + ["0"])
         for i, c in enumerate(configs[1:]):
             current = step_factory(c, context, action_path + [str(i + 1)], previous_step=current)

src/anemoi/datasets/create/input/repeated_dates.py

@@ -12,12 +12,11 @@
 from collections import defaultdict
 
 import numpy as np
+from anemoi.transform.fields import new_field_with_valid_datetime
+from anemoi.transform.fields import new_fieldlist_from_list
 from anemoi.utils.dates import as_datetime
 from anemoi.utils.dates import frequency_to_timedelta
 
-from anemoi.datasets.fields import FieldArray
-from anemoi.datasets.fields import NewValidDateTimeField
-
 from .action import Action
 from .action import action_factory
 from .join import JoinResult
@@ -116,9 +115,10 @@ def transform(self, group_of_dates):
 
 
 class DateMapperClimatology(DateMapper):
-    def __init__(self, source, year, day):
+    def __init__(self, source, year, day, hour=None):
         self.year = year
         self.day = day
+        self.hour = hour
 
     def transform(self, group_of_dates):
         from anemoi.datasets.dates.groups import GroupOfDates
@@ -130,6 +130,8 @@ def transform(self, group_of_dates):
         new_dates = defaultdict(list)
         for date in dates:
             new_date = date.replace(year=self.year, day=self.day)
+            if self.hour is not None:
+                new_date = new_date.replace(hour=self.hour, minute=0, second=0)
             new_dates[new_date].append(date)
 
         for date, dates in new_dates.items():
@@ -185,9 +187,12 @@ def datasource(self):
 
         for field in self.source_results.datasource:
             for date in self.original_group_of_dates:
-                result.append(NewValidDateTimeField(field, date))
+                result.append(new_field_with_valid_datetime(field, date))
+
+        if not result:
+            raise ValueError("repeated_dates: no input data found")
 
-        return FieldArray(result)
+        return new_fieldlist_from_list(result)
 
 
 class RepeatedDatesAction(Action):

src/anemoi/datasets/create/input/result.py

@@ -15,16 +15,10 @@
 from functools import cached_property
 
 import numpy as np
-from anemoi.utils.dates import as_datetime as as_datetime
-from anemoi.utils.dates import frequency_to_timedelta as frequency_to_timedelta
 from anemoi.utils.humanize import seconds_to_human
 from anemoi.utils.humanize import shorten_list
 from earthkit.data.core.order import build_remapping
 
-from anemoi.datasets.dates import DatesProvider as DatesProvider
-from anemoi.datasets.fields import FieldArray as FieldArray
-from anemoi.datasets.fields import NewValidDateTimeField as NewValidDateTimeField
-
 from .trace import trace
 from .trace import trace_datasource
 

src/anemoi/datasets/create/input/step.py

@@ -10,13 +10,6 @@
 import logging
 from copy import deepcopy
 
-from anemoi.utils.dates import as_datetime as as_datetime
-from anemoi.utils.dates import frequency_to_timedelta as frequency_to_timedelta
-
-from anemoi.datasets.dates import DatesProvider as DatesProvider
-from anemoi.datasets.fields import FieldArray as FieldArray
-from anemoi.datasets.fields import NewValidDateTimeField as NewValidDateTimeField
-
 from .action import Action
 from .context import Context
 from .misc import is_function

src/anemoi/datasets/dates/groups.py

@@ -28,7 +28,7 @@ def __init__(self, dates, provider, partial_ok=False):
         assert isinstance(provider, DatesProvider), type(provider)
         assert isinstance(dates, list)
 
-        self.dates = dates
+        self.dates = [as_datetime(_) for _ in dates]
         self.provider = provider
         self.partial_ok = partial_ok