interface: cache API data

Signed-off-by: Álvaro Fernández Rojas <noltari@gmail.com>

.gitignore

@@ -1,5 +1,6 @@
 *.egg-info
 *.gif
+*.json
 *.patch
 *.pyc
 build

aemet_opendata/const.py

@@ -113,6 +113,8 @@
 AOD_WIND_SPEED: Final[str] = "wind-speed"
 AOD_WIND_SPEED_MAX: Final[str] = "wind-speed-max"
 
+API_CALL_FILE_EXTENSION: Final[str] = ".json"
+API_CALL_DATA_TIMEOUT_DEF: Final[timedelta] = timedelta(hours=1)
 API_HDR_REQ_COUNT: Final[str] = "Remaining-request-count"
 API_ID_PFX: Final[str] = "id"
 API_MIN_STATION_DISTANCE_KM: Final[int] = 40
@@ -132,6 +134,7 @@
 ATTR_DATA: Final[str] = "data"
 ATTR_DISTANCE: Final[str] = "distance"
 ATTR_RESPONSE: Final[str] = "response"
+ATTR_TIMESTAMP: Final[str] = "timestamp"
 ATTR_TYPE: Final[str] = "type"
 
 CONTENT_TYPE_IMG: Final[str] = "image/"

aemet_opendata/helpers.py

@@ -1,14 +1,28 @@
 """AEMET OpenData Helpers."""
 
+import base64
 from datetime import datetime
+import json
+import re
 from typing import Any
+import unicodedata
 from zoneinfo import ZoneInfo
 
 from .const import API_ID_PFX, CONTENT_TYPE_IMG
 
 TZ_UTC = ZoneInfo("UTC")
 
 
+class BytesEncoder(json.JSONEncoder):
+    """JSON Bytes Encoder class."""
+
+    def default(self, o: Any) -> Any:
+        """JSON default encoder function."""
+        if isinstance(o, bytes):
+            return base64.b64encode(o).decode("utf-8")
+        return super().default(o)
+
+
 def dict_nested_value(data: dict[str, Any] | None, keys: list[str] | None) -> Any:
     """Get value from dict with nested keys."""
     if keys is None or len(keys) == 0:
@@ -19,9 +33,12 @@ def dict_nested_value(data: dict[str, Any] | None, keys: list[str] | None) -> An
     return data
 
 
-def get_current_datetime(tz: ZoneInfo = TZ_UTC) -> datetime:
+def get_current_datetime(tz: ZoneInfo = TZ_UTC, replace: bool = True) -> datetime:
     """Return current datetime in UTC."""
-    return datetime.now(tz=tz).replace(minute=0, second=0, microsecond=0)
+    cur_dt = datetime.now(tz=tz)
+    if replace:
+        cur_dt.replace(minute=0, second=0, microsecond=0)
+    return cur_dt
 
 
 def split_coordinate(coordinate: str) -> str:
@@ -57,6 +74,20 @@ def parse_town_code(town_id: str) -> str:
     return town_id
 
 
+def slugify(value: str, allow_unicode: bool = False) -> str:
+    """Convert string to a valid file name."""
+    if allow_unicode:
+        value = unicodedata.normalize("NFKC", value)
+    else:
+        value = (
+            unicodedata.normalize("NFKD", value)
+            .encode("ascii", "ignore")
+            .decode("ascii")
+        )
+    value = re.sub(r"[^\w\s]", "-", value.lower())
+    return re.sub(r"[-\s]+", "-", value).strip("-_")
+
+
 def timezone_from_coords(coords: tuple[float, float]) -> ZoneInfo:
     """Convert coordinates to timezone."""
     if coords[0] < 32 and coords[1] < -11.5:

aemet_opendata/interface.py

@@ -2,9 +2,13 @@
 
 import asyncio
 from asyncio import Lock, Semaphore
+import base64
 from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
+import json
 import logging
-from typing import Any, cast
+import os
+from typing import Any, Final, cast
 
 from aiohttp import ClientError, ClientSession, ClientTimeout
 from aiohttp.client_reqrep import ClientResponse
@@ -41,6 +45,8 @@
     AOD_WIND_DIRECTION,
     AOD_WIND_SPEED,
     AOD_WIND_SPEED_MAX,
+    API_CALL_DATA_TIMEOUT_DEF,
+    API_CALL_FILE_EXTENSION,
     API_HDR_REQ_COUNT,
     API_MIN_STATION_DISTANCE_KM,
     API_MIN_TOWN_DISTANCE_KM,
@@ -49,6 +55,7 @@
     ATTR_DATA,
     ATTR_DISTANCE,
     ATTR_RESPONSE,
+    ATTR_TIMESTAMP,
     ATTR_TYPE,
     CONTENT_TYPE_IMG,
     HTTP_CALL_TIMEOUT,
@@ -68,13 +75,31 @@
     TooManyRequests,
     TownNotFound,
 )
-from .helpers import get_current_datetime, parse_station_coordinates, parse_town_code
+from .helpers import (
+    BytesEncoder,
+    get_current_datetime,
+    parse_api_timestamp,
+    parse_station_coordinates,
+    parse_town_code,
+    slugify,
+)
 from .station import Station
 from .town import Town
 
 _LOGGER = logging.getLogger(__name__)
 
 
+API_CALL_DATA_TIMEOUT: Final[dict[str, timedelta]] = {
+    "maestro/municipios": timedelta(days=15),
+    "prediccion/especifica/municipio/diaria": timedelta(days=3),
+    "prediccion/especifica/municipio/horaria": timedelta(hours=48),
+    "observacion/convencional/datos/estacion": timedelta(hours=2),
+    "observacion/convencional/todas": timedelta(days=15),
+    "red/radar/nacional": timedelta(hours=6),
+    "red/rayos/mapa": timedelta(hours=6),
+}
+
+
 @dataclass
 class ConnectionOptions:
     """AEMET OpenData API options for connection."""
@@ -86,6 +111,7 @@ class ConnectionOptions:
 class AEMET:
     """Interacts with the AEMET OpenData API."""
 
+    _api_data_dir: str | None
     _api_raw_data: dict[str, Any]
     _api_raw_data_lock: Lock
     _api_semaphore: Semaphore
@@ -104,6 +130,7 @@ def __init__(
         options: ConnectionOptions,
     ) -> None:
         """Init AEMET OpenData API."""
+        self._api_data_dir = None
         self._api_raw_data = {
             RAW_FORECAST_DAILY: {},
             RAW_FORECAST_HOURLY: {},
@@ -125,6 +152,12 @@ def __init__(
         self.station = None
         self.town = None
 
+    def set_api_data_dir(self, data_dir: str) -> None:
+        """Set API data directory."""
+        if not os.path.exists(data_dir):
+            os.makedirs(data_dir)
+        self._api_data_dir = data_dir
+
     async def set_api_raw_data(self, key: str, subkey: str | None, data: Any) -> None:
         """Save API raw data if not empty."""
         if data is not None:
@@ -134,7 +167,7 @@ async def set_api_raw_data(self, key: str, subkey: str | None, data: Any) -> Non
                 else:
                     self._api_raw_data[key][subkey] = data
 
-    async def api_call(self, cmd: str, fetch_data: bool = False) -> dict[str, Any]:
+    async def _api_call(self, cmd: str, fetch_data: bool = False) -> dict[str, Any]:
         """Perform Rest API call."""
         _LOGGER.debug("api_call: cmd=%s", cmd)
 
@@ -151,6 +184,8 @@ async def api_call(self, cmd: str, fetch_data: bool = False) -> dict[str, Any]:
             except ClientError as err:
                 raise AemetError(err) from err
 
+            cur_dt = get_current_datetime(replace=False)
+
             req_count = resp.headers.get(API_HDR_REQ_COUNT)
             if req_count is not None:
                 await self.set_api_raw_data(RAW_REQ_COUNT, None, req_count)
@@ -185,18 +220,88 @@ async def api_call(self, cmd: str, fetch_data: bool = False) -> dict[str, Any]:
         json_response = cast(dict[str, Any], resp_json)
         if fetch_data and AEMET_ATTR_DATA in json_response:
             data = await self.api_data(json_response[AEMET_ATTR_DATA])
-            if data:
-                json_response = {
-                    ATTR_RESPONSE: json_response,
-                    ATTR_DATA: data,
-                }
-        if isinstance(json_response, list):
             json_response = {
-                ATTR_DATA: json_response,
+                ATTR_RESPONSE: json_response,
+                ATTR_DATA: data,
             }
 
+        json_response[ATTR_TIMESTAMP] = cur_dt.isoformat()
+
         return json_response
 
+    async def api_call(self, cmd: str, fetch_data: bool = False) -> dict[str, Any]:
+        """Provide data from API or file."""
+        json_data: dict[str, Any] | None
+
+        try:
+            json_data = await self._api_call(cmd, fetch_data)
+        except AemetError as err:
+            json_data = self.api_call_load(cmd)
+            if json_data is None:
+                raise err
+            _LOGGER.error(err)
+        else:
+            self.api_call_save(cmd, json_data)
+
+        return json_data
+
+    def api_call_load(self, cmd: str) -> dict[str, Any] | None:
+        """Load API call from file."""
+        json_data: dict[str, Any] | None = None
+
+        if self._api_data_dir is None:
+            return None
+
+        file_name = slugify(cmd) + API_CALL_FILE_EXTENSION
+        file_path = os.path.join(self._api_data_dir, file_name)
+        if not os.path.isfile(file_path):
+            return None
+
+        data_timeout = API_CALL_DATA_TIMEOUT_DEF
+        for key, val in API_CALL_DATA_TIMEOUT.items():
+            if cmd.startswith(key):
+                data_timeout = val
+                break
+
+        _LOGGER.info('Loading cmd=%s from "%s"...', cmd, file_name)
+
+        with open(file_path, "r", encoding="utf-8") as file:
+            file_data = file.read()
+            json_data = json.loads(file_data)
+            file.close()
+
+        json_data = cast(dict[str, Any], json_data)
+
+        file_isotime = json_data.get(ATTR_TIMESTAMP)
+        if file_isotime is not None:
+            file_datetime = parse_api_timestamp(file_isotime)
+        else:
+            file_mtime = os.path.getmtime(file_path)
+            file_datetime = datetime.fromtimestamp(file_mtime, tz=timezone.utc)
+
+        cur_datetime = get_current_datetime(replace=False)
+        if cur_datetime - file_datetime > data_timeout:
+            return None
+
+        json_attr_data = json_data.get(ATTR_DATA, {})
+        if isinstance(json_attr_data, dict):
+            json_bytes = json_attr_data.get(ATTR_BYTES)
+            if json_bytes is not None:
+                json_data[ATTR_DATA][ATTR_BYTES] = base64.b64decode(json_bytes)
+
+        return json_data
+
+    def api_call_save(self, cmd: str, json_data: dict[str, Any]) -> None:
+        """Save API call to file."""
+        if self._api_data_dir is None:
+            return
+
+        file_name = slugify(cmd) + API_CALL_FILE_EXTENSION
+        file_path = os.path.join(self._api_data_dir, file_name)
+        with open(file_path, "w", encoding="utf-8") as file:
+            file.write(json.dumps(json_data, cls=BytesEncoder))
+            file.close()
+
     async def api_data(self, url: str) -> Any:
         """Fetch API data."""
         _LOGGER.debug("api_data: url=%s", url)

examples/_secrets.py

@@ -3,6 +3,7 @@
 from aemet_opendata.interface import ConnectionOptions
 
 AEMET_COORDS = (40.3049863, -3.7550013)
+AEMET_DATA_DIR = "api-data"
 AEMET_OPTIONS = ConnectionOptions(
     api_key="MY_API_KEY",
     station_data=True,

examples/basic.py

@@ -4,7 +4,7 @@
 import timeit
 
 from _common import json_dumps
-from _secrets import AEMET_COORDS, AEMET_OPTIONS
+from _secrets import AEMET_COORDS, AEMET_DATA_DIR, AEMET_OPTIONS
 import aiohttp
 
 from aemet_opendata.exceptions import ApiError, AuthError, TooManyRequests, TownNotFound
@@ -17,6 +17,8 @@ async def main():
     async with aiohttp.ClientSession() as aiohttp_session:
         client = AEMET(aiohttp_session, AEMET_OPTIONS)
 
+        client.set_api_data_dir(AEMET_DATA_DIR)
+
         try:
             select_start = timeit.default_timer()
             await client.select_coordinates(AEMET_COORDS[0], AEMET_COORDS[1])

examples/get-lightnings-map.py

@@ -3,7 +3,7 @@
 import asyncio
 import timeit
 
-from _secrets import AEMET_OPTIONS
+from _secrets import AEMET_DATA_DIR, AEMET_OPTIONS
 import aiohttp
 
 from aemet_opendata.const import ATTR_BYTES, ATTR_DATA, ATTR_TYPE
@@ -18,6 +18,8 @@ async def main():
     async with aiohttp.ClientSession() as aiohttp_session:
         client = AEMET(aiohttp_session, AEMET_OPTIONS)
 
+        client.set_api_data_dir(AEMET_DATA_DIR)
+
         try:
             api_start = timeit.default_timer()
             lightnings_map = await client.get_lightnings_map()

examples/get-radar-map.py

@@ -3,7 +3,7 @@
 import asyncio
 import timeit
 
-from _secrets import AEMET_OPTIONS
+from _secrets import AEMET_DATA_DIR, AEMET_OPTIONS
 import aiohttp
 
 from aemet_opendata.const import ATTR_BYTES, ATTR_DATA, ATTR_TYPE
@@ -18,6 +18,8 @@ async def main():
     async with aiohttp.ClientSession() as aiohttp_session:
         client = AEMET(aiohttp_session, AEMET_OPTIONS)
 
+        client.set_api_data_dir(AEMET_DATA_DIR)
+
         try:
             api_start = timeit.default_timer()
             national_radar = await client.get_radar_map()

examples/get-town-by-coords-precise.py

@@ -4,7 +4,7 @@
 import timeit
 
 from _common import json_dumps
-from _secrets import AEMET_COORDS, AEMET_OPTIONS
+from _secrets import AEMET_COORDS, AEMET_DATA_DIR, AEMET_OPTIONS
 import aiohttp
 
 from aemet_opendata.exceptions import AuthError
@@ -17,6 +17,7 @@ async def main():
     async with aiohttp.ClientSession() as aiohttp_session:
         client = AEMET(aiohttp_session, AEMET_OPTIONS)
 
+        client.set_api_data_dir(AEMET_DATA_DIR)
         client.distance_high_precision(True)
 
         try:

examples/get-town-by-coords.py

@@ -4,7 +4,7 @@
 import timeit
 
 from _common import json_dumps
-from _secrets import AEMET_COORDS, AEMET_OPTIONS
+from _secrets import AEMET_COORDS, AEMET_DATA_DIR, AEMET_OPTIONS
 import aiohttp
 
 from aemet_opendata.exceptions import AuthError
@@ -17,6 +17,8 @@ async def main():
     async with aiohttp.ClientSession() as aiohttp_session:
         client = AEMET(aiohttp_session, AEMET_OPTIONS)
 
+        client.set_api_data_dir(AEMET_DATA_DIR)
+
         try:
             get_town_start = timeit.default_timer()
             town = await client.get_town_by_coordinates(