tutorials/00-key-concepts.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tempfile\n",
    "import time\n",
    "from os import path\n",
    "\n",
    "import pandas as pd\n",
    "import tstore\n",
    "from tstore.tslong.pandas import open_tslong, to_tstore"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1",
   "metadata": {},
   "source": [
    "# Key Concepts\n",
    "\n",
    "Let us consider the following dataset of observations from a set of meteorological stations obtained from the Netatmo Weather API (only non-commercial uses of this data are acceptable, see [the Netatmo APIs terms of use](https://dev.netatmo.com/legal)):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2",
   "metadata": {
    "tags": [
     "parameters"
    ]
   },
   "outputs": [],
   "source": [
    "# ts_df_filepath = \"../data/interim/netatmo-lausanne-aug-21/ts-df.csv\"\n",
    "ts_df_filepath = \"../data/interim/netatmo-lausanne-aug-21/ts-df-clean.csv\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3",
   "metadata": {},
   "source": [
    "The time series file contains the following columns:\n",
    "- `variable`: the variable measured (e.g., temperature, humidity...)\n",
    "- `id`: the identifier of the station\n",
    "- `time`: the time of the measurement\n",
    "- `value`: the value of the measurement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>variable</th>\n",
       "      <th>id</th>\n",
       "      <th>time</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>humidity</td>\n",
       "      <td>70:ee:50:00:10:f0</td>\n",
       "      <td>2021-08-12 13:00:00</td>\n",
       "      <td>50.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>humidity</td>\n",
       "      <td>70:ee:50:00:10:f0</td>\n",
       "      <td>2021-08-12 14:00:00</td>\n",
       "      <td>49.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>humidity</td>\n",
       "      <td>70:ee:50:00:10:f0</td>\n",
       "      <td>2021-08-12 15:00:00</td>\n",
       "      <td>50.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>humidity</td>\n",
       "      <td>70:ee:50:00:10:f0</td>\n",
       "      <td>2021-08-12 16:00:00</td>\n",
       "      <td>49.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>humidity</td>\n",
       "      <td>70:ee:50:00:10:f0</td>\n",
       "      <td>2021-08-12 17:00:00</td>\n",
       "      <td>53.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>381071</th>\n",
       "      <td>wind_strength</td>\n",
       "      <td>70:ee:50:7f:98:de</td>\n",
       "      <td>2021-08-23 22:00:00</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>381072</th>\n",
       "      <td>wind_strength</td>\n",
       "      <td>70:ee:50:7f:98:de</td>\n",
       "      <td>2021-08-25 01:00:00</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>381073</th>\n",
       "      <td>wind_strength</td>\n",
       "      <td>70:ee:50:7f:98:de</td>\n",
       "      <td>2021-08-25 02:00:00</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>381074</th>\n",
       "      <td>wind_strength</td>\n",
       "      <td>70:ee:50:7f:98:de</td>\n",
       "      <td>2021-08-25 03:00:00</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>381075</th>\n",
       "      <td>wind_strength</td>\n",
       "      <td>70:ee:50:7f:98:de</td>\n",
       "      <td>2021-08-25 04:00:00</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>381076 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             variable                 id                time  value\n",
       "0            humidity  70:ee:50:00:10:f0 2021-08-12 13:00:00   50.0\n",
       "1            humidity  70:ee:50:00:10:f0 2021-08-12 14:00:00   49.0\n",
       "2            humidity  70:ee:50:00:10:f0 2021-08-12 15:00:00   50.0\n",
       "3            humidity  70:ee:50:00:10:f0 2021-08-12 16:00:00   49.0\n",
       "4            humidity  70:ee:50:00:10:f0 2021-08-12 17:00:00   53.0\n",
       "...               ...                ...                 ...    ...\n",
       "381071  wind_strength  70:ee:50:7f:98:de 2021-08-23 22:00:00    2.0\n",
       "381072  wind_strength  70:ee:50:7f:98:de 2021-08-25 01:00:00    3.0\n",
       "381073  wind_strength  70:ee:50:7f:98:de 2021-08-25 02:00:00    3.0\n",
       "381074  wind_strength  70:ee:50:7f:98:de 2021-08-25 03:00:00    2.0\n",
       "381075  wind_strength  70:ee:50:7f:98:de 2021-08-25 04:00:00    3.0\n",
       "\n",
       "[381076 rows x 4 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(ts_df_filepath)\n",
    "df[\"time\"] = pd.to_datetime(df[\"time\"])\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5",
   "metadata": {},
   "source": [
    "## Time series\n",
    "\n",
    "A time series is a sequence of data indexed by timestamps. In our case, we have multiple time series, each corresponding to a meteorological variable measured by a station module. Consider data for the following station"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>variable</th>\n",
       "      <th>id</th>\n",
       "      <th>time</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>humidity</td>\n",
       "      <td>70:ee:50:00:10:f0</td>\n",
       "      <td>2021-08-12 13:00:00</td>\n",
       "      <td>50.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>humidity</td>\n",
       "      <td>70:ee:50:00:10:f0</td>\n",
       "      <td>2021-08-12 14:00:00</td>\n",
       "      <td>49.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>humidity</td>\n",
       "      <td>70:ee:50:00:10:f0</td>\n",
       "      <td>2021-08-12 15:00:00</td>\n",
       "      <td>50.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>humidity</td>\n",
       "      <td>70:ee:50:00:10:f0</td>\n",
       "      <td>2021-08-12 16:00:00</td>\n",
       "      <td>49.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>humidity</td>\n",
       "      <td>70:ee:50:00:10:f0</td>\n",
       "      <td>2021-08-12 17:00:00</td>\n",
       "      <td>53.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>234565</th>\n",
       "      <td>temperature</td>\n",
       "      <td>70:ee:50:00:10:f0</td>\n",
       "      <td>2021-08-25 01:00:00</td>\n",
       "      <td>11.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>234566</th>\n",
       "      <td>temperature</td>\n",
       "      <td>70:ee:50:00:10:f0</td>\n",
       "      <td>2021-08-25 02:00:00</td>\n",
       "      <td>10.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>234567</th>\n",
       "      <td>temperature</td>\n",
       "      <td>70:ee:50:00:10:f0</td>\n",
       "      <td>2021-08-25 05:00:00</td>\n",
       "      <td>11.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>234568</th>\n",
       "      <td>temperature</td>\n",
       "      <td>70:ee:50:00:10:f0</td>\n",
       "      <td>2021-08-25 06:00:00</td>\n",
       "      <td>13.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>234569</th>\n",
       "      <td>temperature</td>\n",
       "      <td>70:ee:50:00:10:f0</td>\n",
       "      <td>2021-08-25 07:00:00</td>\n",
       "      <td>15.4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>741 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           variable                 id                time  value\n",
       "0          humidity  70:ee:50:00:10:f0 2021-08-12 13:00:00   50.0\n",
       "1          humidity  70:ee:50:00:10:f0 2021-08-12 14:00:00   49.0\n",
       "2          humidity  70:ee:50:00:10:f0 2021-08-12 15:00:00   50.0\n",
       "3          humidity  70:ee:50:00:10:f0 2021-08-12 16:00:00   49.0\n",
       "4          humidity  70:ee:50:00:10:f0 2021-08-12 17:00:00   53.0\n",
       "...             ...                ...                 ...    ...\n",
       "234565  temperature  70:ee:50:00:10:f0 2021-08-25 01:00:00   11.4\n",
       "234566  temperature  70:ee:50:00:10:f0 2021-08-25 02:00:00   10.9\n",
       "234567  temperature  70:ee:50:00:10:f0 2021-08-25 05:00:00   11.4\n",
       "234568  temperature  70:ee:50:00:10:f0 2021-08-25 06:00:00   13.0\n",
       "234569  temperature  70:ee:50:00:10:f0 2021-08-25 07:00:00   15.4\n",
       "\n",
       "[741 rows x 4 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "station_id = \"70:ee:50:00:10:f0\"\n",
    "df.loc[df[\"id\"] == station_id]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7",
   "metadata": {},
   "source": [
    "In order to have a proper time series data structure, we can pivot the data to have each variable as a column and each observation as a row (see more details in the [data forms section below](#data-forms)):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>variable</th>\n",
       "      <th>humidity</th>\n",
       "      <th>pressure</th>\n",
       "      <th>temperature</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>time</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2021-08-12 13:00:00</th>\n",
       "      <td>50.0</td>\n",
       "      <td>1023.7</td>\n",
       "      <td>28.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-12 14:00:00</th>\n",
       "      <td>49.0</td>\n",
       "      <td>1023.3</td>\n",
       "      <td>29.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-12 15:00:00</th>\n",
       "      <td>50.0</td>\n",
       "      <td>1023.3</td>\n",
       "      <td>29.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-12 16:00:00</th>\n",
       "      <td>49.0</td>\n",
       "      <td>1022.2</td>\n",
       "      <td>29.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-12 17:00:00</th>\n",
       "      <td>53.0</td>\n",
       "      <td>1023.3</td>\n",
       "      <td>27.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-25 01:00:00</th>\n",
       "      <td>80.0</td>\n",
       "      <td>1019.3</td>\n",
       "      <td>11.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-25 02:00:00</th>\n",
       "      <td>81.0</td>\n",
       "      <td>1019.2</td>\n",
       "      <td>10.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-25 05:00:00</th>\n",
       "      <td>83.0</td>\n",
       "      <td>1019.2</td>\n",
       "      <td>11.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-25 06:00:00</th>\n",
       "      <td>83.0</td>\n",
       "      <td>1019.2</td>\n",
       "      <td>13.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-25 07:00:00</th>\n",
       "      <td>82.0</td>\n",
       "      <td>1019.6</td>\n",
       "      <td>15.4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>247 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "variable             humidity  pressure  temperature\n",
       "time                                                \n",
       "2021-08-12 13:00:00      50.0    1023.7         28.1\n",
       "2021-08-12 14:00:00      49.0    1023.3         29.5\n",
       "2021-08-12 15:00:00      50.0    1023.3         29.4\n",
       "2021-08-12 16:00:00      49.0    1022.2         29.2\n",
       "2021-08-12 17:00:00      53.0    1023.3         27.9\n",
       "...                       ...       ...          ...\n",
       "2021-08-25 01:00:00      80.0    1019.3         11.4\n",
       "2021-08-25 02:00:00      81.0    1019.2         10.9\n",
       "2021-08-25 05:00:00      83.0    1019.2         11.4\n",
       "2021-08-25 06:00:00      83.0    1019.2         13.0\n",
       "2021-08-25 07:00:00      82.0    1019.6         15.4\n",
       "\n",
       "[247 rows x 3 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts_df = df.loc[df[\"id\"] == station_id].pivot_table(\n",
    "    index=\"time\",\n",
    "    columns=\"variable\",\n",
    "    values=\"value\",\n",
    ")\n",
    "ts_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9",
   "metadata": {},
   "source": [
    "The time series above can be encapsulated in a `TS` object:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TS[start=2021-08-12 13:00:00,end=2021-08-25 07:00:00;shape=(247, 3)]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts = tstore.TS(ts_df)\n",
    "ts"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "11",
   "metadata": {},
   "source": [
    "Why would we need such an encapsulation? Let us get the data for two stations and pivot it to have each variable *for each station* as a column and each observation as a row:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th>70:ee:50:00:c7:72</th>\n",
       "      <th>70:ee:50:15:ea:9e</th>\n",
       "      <th>70:ee:50:00:c7:72</th>\n",
       "      <th colspan=\"2\" halign=\"left\">70:ee:50:15:ea:9e</th>\n",
       "      <th>70:ee:50:00:c7:72</th>\n",
       "      <th>70:ee:50:15:ea:9e</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>variable</th>\n",
       "      <th>humidity</th>\n",
       "      <th>humidity</th>\n",
       "      <th>pressure</th>\n",
       "      <th>pressure</th>\n",
       "      <th>rain_live</th>\n",
       "      <th>temperature</th>\n",
       "      <th>temperature</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>time</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2021-08-12 12:00:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>49.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1019.8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>31.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-12 13:00:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>52.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1019.6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>29.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-12 14:00:00</th>\n",
       "      <td>53.0</td>\n",
       "      <td>51.0</td>\n",
       "      <td>1024.3</td>\n",
       "      <td>1019.6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>29.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-12 15:00:00</th>\n",
       "      <td>51.0</td>\n",
       "      <td>54.0</td>\n",
       "      <td>1023.9</td>\n",
       "      <td>1019.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>30.2</td>\n",
       "      <td>29.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-12 16:00:00</th>\n",
       "      <td>53.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>1023.1</td>\n",
       "      <td>1018.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>30.2</td>\n",
       "      <td>30.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-25 05:00:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>90.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1015.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-25 06:00:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>90.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1015.7</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-25 07:00:00</th>\n",
       "      <td>NaN</td>\n",
       "      <td>90.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1015.7</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-25 08:00:00</th>\n",
       "      <td>77.0</td>\n",
       "      <td>89.0</td>\n",
       "      <td>1021.8</td>\n",
       "      <td>1016.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15.4</td>\n",
       "      <td>13.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-25 09:00:00</th>\n",
       "      <td>76.0</td>\n",
       "      <td>86.0</td>\n",
       "      <td>1021.9</td>\n",
       "      <td>1016.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>16.2</td>\n",
       "      <td>14.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>223 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "id                  70:ee:50:00:c7:72 70:ee:50:15:ea:9e 70:ee:50:00:c7:72  \\\n",
       "variable                     humidity          humidity          pressure   \n",
       "time                                                                        \n",
       "2021-08-12 12:00:00               NaN              49.0               NaN   \n",
       "2021-08-12 13:00:00               NaN              52.0               NaN   \n",
       "2021-08-12 14:00:00              53.0              51.0            1024.3   \n",
       "2021-08-12 15:00:00              51.0              54.0            1023.9   \n",
       "2021-08-12 16:00:00              53.0              52.0            1023.1   \n",
       "...                               ...               ...               ...   \n",
       "2021-08-25 05:00:00               NaN              90.0               NaN   \n",
       "2021-08-25 06:00:00               NaN              90.0               NaN   \n",
       "2021-08-25 07:00:00               NaN              90.0               NaN   \n",
       "2021-08-25 08:00:00              77.0              89.0            1021.8   \n",
       "2021-08-25 09:00:00              76.0              86.0            1021.9   \n",
       "\n",
       "id                  70:ee:50:15:ea:9e           70:ee:50:00:c7:72  \\\n",
       "variable                     pressure rain_live       temperature   \n",
       "time                                                                \n",
       "2021-08-12 12:00:00            1019.8       0.0               NaN   \n",
       "2021-08-12 13:00:00            1019.6       0.0               NaN   \n",
       "2021-08-12 14:00:00            1019.6       0.0              30.0   \n",
       "2021-08-12 15:00:00            1019.2       0.0              30.2   \n",
       "2021-08-12 16:00:00            1018.3       0.0              30.2   \n",
       "...                               ...       ...               ...   \n",
       "2021-08-25 05:00:00            1015.5       0.0               NaN   \n",
       "2021-08-25 06:00:00            1015.7       0.0               NaN   \n",
       "2021-08-25 07:00:00            1015.7       0.0               NaN   \n",
       "2021-08-25 08:00:00            1016.0       0.0              15.4   \n",
       "2021-08-25 09:00:00            1016.1       0.0              16.2   \n",
       "\n",
       "id                  70:ee:50:15:ea:9e  \n",
       "variable                  temperature  \n",
       "time                                   \n",
       "2021-08-12 12:00:00              31.6  \n",
       "2021-08-12 13:00:00              29.9  \n",
       "2021-08-12 14:00:00              29.4  \n",
       "2021-08-12 15:00:00              29.7  \n",
       "2021-08-12 16:00:00              30.4  \n",
       "...                               ...  \n",
       "2021-08-25 05:00:00              11.5  \n",
       "2021-08-25 06:00:00              11.9  \n",
       "2021-08-25 07:00:00              12.4  \n",
       "2021-08-25 08:00:00              13.1  \n",
       "2021-08-25 09:00:00              14.0  \n",
       "\n",
       "[223 rows x 7 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "station_ids = [\"70:ee:50:00:c7:72\", \"70:ee:50:15:ea:9e\"]\n",
    "ts_df = df[df[\"id\"].isin(station_ids)].pivot_table(\n",
    "    index=\"time\",\n",
    "    columns=[\"id\", \"variable\"],\n",
    "    values=\"value\",\n",
    ")\n",
    "ts_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "13",
   "metadata": {},
   "source": [
    "Note now that each station has several missing values, therefore aligning the time index would result in many nan values:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id                 variable   \n",
       "70:ee:50:00:c7:72  humidity        45\n",
       "70:ee:50:15:ea:9e  humidity       105\n",
       "70:ee:50:00:c7:72  pressure        45\n",
       "70:ee:50:15:ea:9e  pressure       105\n",
       "                   rain_live      105\n",
       "70:ee:50:00:c7:72  temperature     45\n",
       "70:ee:50:15:ea:9e  temperature    105\n",
       "dtype: int64"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts_df.isna().sum(axis=\"rows\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15",
   "metadata": {},
   "source": [
    "Instead, we can create a `TS` object for each station and put them in a pandas object. Let us first transform it into a long data frame:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "16",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>variable</th>\n",
       "      <th>humidity</th>\n",
       "      <th>pressure</th>\n",
       "      <th>rain_live</th>\n",
       "      <th>temperature</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th>time</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">70:ee:50:00:c7:72</th>\n",
       "      <th>2021-08-12 14:00:00</th>\n",
       "      <td>53.0</td>\n",
       "      <td>1024.3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>30.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-12 15:00:00</th>\n",
       "      <td>51.0</td>\n",
       "      <td>1023.9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>30.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-12 16:00:00</th>\n",
       "      <td>53.0</td>\n",
       "      <td>1023.1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>30.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-12 17:00:00</th>\n",
       "      <td>55.0</td>\n",
       "      <td>1023.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>29.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-12 18:00:00</th>\n",
       "      <td>61.0</td>\n",
       "      <td>1023.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>27.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">70:ee:50:15:ea:9e</th>\n",
       "      <th>2021-08-25 05:00:00</th>\n",
       "      <td>90.0</td>\n",
       "      <td>1015.5</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-25 06:00:00</th>\n",
       "      <td>90.0</td>\n",
       "      <td>1015.7</td>\n",
       "      <td>0.0</td>\n",
       "      <td>11.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-25 07:00:00</th>\n",
       "      <td>90.0</td>\n",
       "      <td>1015.7</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-25 08:00:00</th>\n",
       "      <td>89.0</td>\n",
       "      <td>1016.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2021-08-25 09:00:00</th>\n",
       "      <td>86.0</td>\n",
       "      <td>1016.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>14.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>296 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "variable                               humidity  pressure  rain_live  \\\n",
       "id                time                                                 \n",
       "70:ee:50:00:c7:72 2021-08-12 14:00:00      53.0    1024.3        NaN   \n",
       "                  2021-08-12 15:00:00      51.0    1023.9        NaN   \n",
       "                  2021-08-12 16:00:00      53.0    1023.1        NaN   \n",
       "                  2021-08-12 17:00:00      55.0    1023.7        NaN   \n",
       "                  2021-08-12 18:00:00      61.0    1023.5        NaN   \n",
       "...                                         ...       ...        ...   \n",
       "70:ee:50:15:ea:9e 2021-08-25 05:00:00      90.0    1015.5        0.0   \n",
       "                  2021-08-25 06:00:00      90.0    1015.7        0.0   \n",
       "                  2021-08-25 07:00:00      90.0    1015.7        0.0   \n",
       "                  2021-08-25 08:00:00      89.0    1016.0        0.0   \n",
       "                  2021-08-25 09:00:00      86.0    1016.1        0.0   \n",
       "\n",
       "variable                               temperature  \n",
       "id                time                              \n",
       "70:ee:50:00:c7:72 2021-08-12 14:00:00         30.0  \n",
       "                  2021-08-12 15:00:00         30.2  \n",
       "                  2021-08-12 16:00:00         30.2  \n",
       "                  2021-08-12 17:00:00         29.0  \n",
       "                  2021-08-12 18:00:00         27.4  \n",
       "...                                            ...  \n",
       "70:ee:50:15:ea:9e 2021-08-25 05:00:00         11.5  \n",
       "                  2021-08-25 06:00:00         11.9  \n",
       "                  2021-08-25 07:00:00         12.4  \n",
       "                  2021-08-25 08:00:00         13.1  \n",
       "                  2021-08-25 09:00:00         14.0  \n",
       "\n",
       "[296 rows x 4 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "long_ts_df = df[df[\"id\"].isin(station_ids)].pivot_table(\n",
    "    index=[\"id\", \"time\"],\n",
    "    columns=\"variable\",\n",
    "    values=\"value\",\n",
    ")\n",
    "long_ts_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17",
   "metadata": {},
   "source": [
    "We can now transform the above data frame into a series of `TS` objects:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "70:ee:50:00:c7:72    TS[start=2021-08-12 14:00:00,end=2021-08-25 09...\n",
       "70:ee:50:15:ea:9e    TS[start=2021-08-12 12:00:00,end=2021-08-25 09...\n",
       "dtype: TS[dask.Series]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts_ser = pd.Series(\n",
    "    tstore.TSArray(\n",
    "        [tstore.TS(long_ts_df.loc[station_id]) for station_id in station_ids],\n",
    "    ),\n",
    "    index=station_ids,\n",
    ")\n",
    "ts_ser"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19",
   "metadata": {},
   "source": [
    "Note that a `TSArray` is [a pandas `ExtensionArray` which is used to support custom data types](https://pandas.pydata.org/docs/reference/api/pandas.api.extensions.ExtensionArray.html), in this case, `TS` objects.\n",
    "\n",
    "## TSDF\n",
    "\n",
    "In some cases columns with different time indexing may share the same identifier. For instance, Netatmo stations can have up to three modules: one for temperature, humidity and pressure, another for wind and a last one for rain. Note that even for the same station, each of this modules can have its own time indexing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>humidity</th>\n",
       "      <th>pressure</th>\n",
       "      <th>rain_live</th>\n",
       "      <th>temperature</th>\n",
       "      <th>wind_angle</th>\n",
       "      <th>wind_strength</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>70:ee:50:00:10:f0</th>\n",
       "      <td>TS[shape=(247,),start=2021-08-12 13:00:00,end=...</td>\n",
       "      <td>TS[shape=(247,),start=2021-08-12 13:00:00,end=...</td>\n",
       "      <td>TS[shape=(0,),start=NaT,end=NaT]</td>\n",
       "      <td>TS[shape=(247,),start=2021-08-12 13:00:00,end=...</td>\n",
       "      <td>TS[shape=(0,),start=NaT,end=NaT]</td>\n",
       "      <td>TS[shape=(0,),start=NaT,end=NaT]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70:ee:50:00:17:8e</th>\n",
       "      <td>TS[shape=(41,),start=2021-08-17 00:00:00,end=2...</td>\n",
       "      <td>TS[shape=(41,),start=2021-08-17 00:00:00,end=2...</td>\n",
       "      <td>TS[shape=(0,),start=NaT,end=NaT]</td>\n",
       "      <td>TS[shape=(41,),start=2021-08-17 00:00:00,end=2...</td>\n",
       "      <td>TS[shape=(0,),start=NaT,end=NaT]</td>\n",
       "      <td>TS[shape=(0,),start=NaT,end=NaT]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70:ee:50:00:20:88</th>\n",
       "      <td>TS[shape=(295,),start=2021-08-12 12:00:00,end=...</td>\n",
       "      <td>TS[shape=(295,),start=2021-08-12 12:00:00,end=...</td>\n",
       "      <td>TS[shape=(295,),start=2021-08-12 12:00:00,end=...</td>\n",
       "      <td>TS[shape=(295,),start=2021-08-12 12:00:00,end=...</td>\n",
       "      <td>TS[shape=(0,),start=NaT,end=NaT]</td>\n",
       "      <td>TS[shape=(0,),start=NaT,end=NaT]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70:ee:50:00:30:2a</th>\n",
       "      <td>TS[shape=(246,),start=2021-08-12 12:00:00,end=...</td>\n",
       "      <td>TS[shape=(246,),start=2021-08-12 12:00:00,end=...</td>\n",
       "      <td>TS[shape=(0,),start=NaT,end=NaT]</td>\n",
       "      <td>TS[shape=(246,),start=2021-08-12 12:00:00,end=...</td>\n",
       "      <td>TS[shape=(0,),start=NaT,end=NaT]</td>\n",
       "      <td>TS[shape=(0,),start=NaT,end=NaT]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70:ee:50:00:36:d0</th>\n",
       "      <td>TS[shape=(178,),start=2021-08-12 13:00:00,end=...</td>\n",
       "      <td>TS[shape=(178,),start=2021-08-12 13:00:00,end=...</td>\n",
       "      <td>TS[shape=(178,),start=2021-08-12 13:00:00,end=...</td>\n",
       "      <td>TS[shape=(178,),start=2021-08-12 13:00:00,end=...</td>\n",
       "      <td>TS[shape=(178,),start=2021-08-12 13:00:00,end=...</td>\n",
       "      <td>TS[shape=(178,),start=2021-08-12 13:00:00,end=...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70:ee:50:7a:c4:fc</th>\n",
       "      <td>TS[shape=(155,),start=2021-08-12 22:00:00,end=...</td>\n",
       "      <td>TS[shape=(155,),start=2021-08-12 22:00:00,end=...</td>\n",
       "      <td>TS[shape=(0,),start=NaT,end=NaT]</td>\n",
       "      <td>TS[shape=(155,),start=2021-08-12 22:00:00,end=...</td>\n",
       "      <td>TS[shape=(0,),start=NaT,end=NaT]</td>\n",
       "      <td>TS[shape=(0,),start=NaT,end=NaT]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70:ee:50:7a:d2:c2</th>\n",
       "      <td>TS[shape=(88,),start=2021-08-12 13:00:00,end=2...</td>\n",
       "      <td>TS[shape=(88,),start=2021-08-12 13:00:00,end=2...</td>\n",
       "      <td>TS[shape=(88,),start=2021-08-12 13:00:00,end=2...</td>\n",
       "      <td>TS[shape=(88,),start=2021-08-12 13:00:00,end=2...</td>\n",
       "      <td>TS[shape=(88,),start=2021-08-12 13:00:00,end=2...</td>\n",
       "      <td>TS[shape=(88,),start=2021-08-12 13:00:00,end=2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70:ee:50:7a:dc:ce</th>\n",
       "      <td>TS[shape=(12,),start=2021-08-15 14:00:00,end=2...</td>\n",
       "      <td>TS[shape=(12,),start=2021-08-15 14:00:00,end=2...</td>\n",
       "      <td>TS[shape=(0,),start=NaT,end=NaT]</td>\n",
       "      <td>TS[shape=(12,),start=2021-08-15 14:00:00,end=2...</td>\n",
       "      <td>TS[shape=(0,),start=NaT,end=NaT]</td>\n",
       "      <td>TS[shape=(0,),start=NaT,end=NaT]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70:ee:50:7a:e3:4a</th>\n",
       "      <td>TS[shape=(211,),start=2021-08-12 12:00:00,end=...</td>\n",
       "      <td>TS[shape=(211,),start=2021-08-12 12:00:00,end=...</td>\n",
       "      <td>TS[shape=(119,),start=2021-08-12 12:00:00,end=...</td>\n",
       "      <td>TS[shape=(211,),start=2021-08-12 12:00:00,end=...</td>\n",
       "      <td>TS[shape=(0,),start=NaT,end=NaT]</td>\n",
       "      <td>TS[shape=(0,),start=NaT,end=NaT]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70:ee:50:7f:98:de</th>\n",
       "      <td>TS[shape=(19,),start=2021-08-13 22:00:00,end=2...</td>\n",
       "      <td>TS[shape=(19,),start=2021-08-13 22:00:00,end=2...</td>\n",
       "      <td>TS[shape=(16,),start=2021-08-13 22:00:00,end=2...</td>\n",
       "      <td>TS[shape=(19,),start=2021-08-13 22:00:00,end=2...</td>\n",
       "      <td>TS[shape=(17,),start=2021-08-13 22:00:00,end=2...</td>\n",
       "      <td>TS[shape=(17,),start=2021-08-13 22:00:00,end=2...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>748 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                            humidity  \\\n",
       "id                                                                     \n",
       "70:ee:50:00:10:f0  TS[shape=(247,),start=2021-08-12 13:00:00,end=...   \n",
       "70:ee:50:00:17:8e  TS[shape=(41,),start=2021-08-17 00:00:00,end=2...   \n",
       "70:ee:50:00:20:88  TS[shape=(295,),start=2021-08-12 12:00:00,end=...   \n",
       "70:ee:50:00:30:2a  TS[shape=(246,),start=2021-08-12 12:00:00,end=...   \n",
       "70:ee:50:00:36:d0  TS[shape=(178,),start=2021-08-12 13:00:00,end=...   \n",
       "...                                                              ...   \n",
       "70:ee:50:7a:c4:fc  TS[shape=(155,),start=2021-08-12 22:00:00,end=...   \n",
       "70:ee:50:7a:d2:c2  TS[shape=(88,),start=2021-08-12 13:00:00,end=2...   \n",
       "70:ee:50:7a:dc:ce  TS[shape=(12,),start=2021-08-15 14:00:00,end=2...   \n",
       "70:ee:50:7a:e3:4a  TS[shape=(211,),start=2021-08-12 12:00:00,end=...   \n",
       "70:ee:50:7f:98:de  TS[shape=(19,),start=2021-08-13 22:00:00,end=2...   \n",
       "\n",
       "                                                            pressure  \\\n",
       "id                                                                     \n",
       "70:ee:50:00:10:f0  TS[shape=(247,),start=2021-08-12 13:00:00,end=...   \n",
       "70:ee:50:00:17:8e  TS[shape=(41,),start=2021-08-17 00:00:00,end=2...   \n",
       "70:ee:50:00:20:88  TS[shape=(295,),start=2021-08-12 12:00:00,end=...   \n",
       "70:ee:50:00:30:2a  TS[shape=(246,),start=2021-08-12 12:00:00,end=...   \n",
       "70:ee:50:00:36:d0  TS[shape=(178,),start=2021-08-12 13:00:00,end=...   \n",
       "...                                                              ...   \n",
       "70:ee:50:7a:c4:fc  TS[shape=(155,),start=2021-08-12 22:00:00,end=...   \n",
       "70:ee:50:7a:d2:c2  TS[shape=(88,),start=2021-08-12 13:00:00,end=2...   \n",
       "70:ee:50:7a:dc:ce  TS[shape=(12,),start=2021-08-15 14:00:00,end=2...   \n",
       "70:ee:50:7a:e3:4a  TS[shape=(211,),start=2021-08-12 12:00:00,end=...   \n",
       "70:ee:50:7f:98:de  TS[shape=(19,),start=2021-08-13 22:00:00,end=2...   \n",
       "\n",
       "                                                           rain_live  \\\n",
       "id                                                                     \n",
       "70:ee:50:00:10:f0                   TS[shape=(0,),start=NaT,end=NaT]   \n",
       "70:ee:50:00:17:8e                   TS[shape=(0,),start=NaT,end=NaT]   \n",
       "70:ee:50:00:20:88  TS[shape=(295,),start=2021-08-12 12:00:00,end=...   \n",
       "70:ee:50:00:30:2a                   TS[shape=(0,),start=NaT,end=NaT]   \n",
       "70:ee:50:00:36:d0  TS[shape=(178,),start=2021-08-12 13:00:00,end=...   \n",
       "...                                                              ...   \n",
       "70:ee:50:7a:c4:fc                   TS[shape=(0,),start=NaT,end=NaT]   \n",
       "70:ee:50:7a:d2:c2  TS[shape=(88,),start=2021-08-12 13:00:00,end=2...   \n",
       "70:ee:50:7a:dc:ce                   TS[shape=(0,),start=NaT,end=NaT]   \n",
       "70:ee:50:7a:e3:4a  TS[shape=(119,),start=2021-08-12 12:00:00,end=...   \n",
       "70:ee:50:7f:98:de  TS[shape=(16,),start=2021-08-13 22:00:00,end=2...   \n",
       "\n",
       "                                                         temperature  \\\n",
       "id                                                                     \n",
       "70:ee:50:00:10:f0  TS[shape=(247,),start=2021-08-12 13:00:00,end=...   \n",
       "70:ee:50:00:17:8e  TS[shape=(41,),start=2021-08-17 00:00:00,end=2...   \n",
       "70:ee:50:00:20:88  TS[shape=(295,),start=2021-08-12 12:00:00,end=...   \n",
       "70:ee:50:00:30:2a  TS[shape=(246,),start=2021-08-12 12:00:00,end=...   \n",
       "70:ee:50:00:36:d0  TS[shape=(178,),start=2021-08-12 13:00:00,end=...   \n",
       "...                                                              ...   \n",
       "70:ee:50:7a:c4:fc  TS[shape=(155,),start=2021-08-12 22:00:00,end=...   \n",
       "70:ee:50:7a:d2:c2  TS[shape=(88,),start=2021-08-12 13:00:00,end=2...   \n",
       "70:ee:50:7a:dc:ce  TS[shape=(12,),start=2021-08-15 14:00:00,end=2...   \n",
       "70:ee:50:7a:e3:4a  TS[shape=(211,),start=2021-08-12 12:00:00,end=...   \n",
       "70:ee:50:7f:98:de  TS[shape=(19,),start=2021-08-13 22:00:00,end=2...   \n",
       "\n",
       "                                                          wind_angle  \\\n",
       "id                                                                     \n",
       "70:ee:50:00:10:f0                   TS[shape=(0,),start=NaT,end=NaT]   \n",
       "70:ee:50:00:17:8e                   TS[shape=(0,),start=NaT,end=NaT]   \n",
       "70:ee:50:00:20:88                   TS[shape=(0,),start=NaT,end=NaT]   \n",
       "70:ee:50:00:30:2a                   TS[shape=(0,),start=NaT,end=NaT]   \n",
       "70:ee:50:00:36:d0  TS[shape=(178,),start=2021-08-12 13:00:00,end=...   \n",
       "...                                                              ...   \n",
       "70:ee:50:7a:c4:fc                   TS[shape=(0,),start=NaT,end=NaT]   \n",
       "70:ee:50:7a:d2:c2  TS[shape=(88,),start=2021-08-12 13:00:00,end=2...   \n",
       "70:ee:50:7a:dc:ce                   TS[shape=(0,),start=NaT,end=NaT]   \n",
       "70:ee:50:7a:e3:4a                   TS[shape=(0,),start=NaT,end=NaT]   \n",
       "70:ee:50:7f:98:de  TS[shape=(17,),start=2021-08-13 22:00:00,end=2...   \n",
       "\n",
       "                                                       wind_strength  \n",
       "id                                                                    \n",
       "70:ee:50:00:10:f0                   TS[shape=(0,),start=NaT,end=NaT]  \n",
       "70:ee:50:00:17:8e                   TS[shape=(0,),start=NaT,end=NaT]  \n",
       "70:ee:50:00:20:88                   TS[shape=(0,),start=NaT,end=NaT]  \n",
       "70:ee:50:00:30:2a                   TS[shape=(0,),start=NaT,end=NaT]  \n",
       "70:ee:50:00:36:d0  TS[shape=(178,),start=2021-08-12 13:00:00,end=...  \n",
       "...                                                              ...  \n",
       "70:ee:50:7a:c4:fc                   TS[shape=(0,),start=NaT,end=NaT]  \n",
       "70:ee:50:7a:d2:c2  TS[shape=(88,),start=2021-08-12 13:00:00,end=2...  \n",
       "70:ee:50:7a:dc:ce                   TS[shape=(0,),start=NaT,end=NaT]  \n",
       "70:ee:50:7a:e3:4a                   TS[shape=(0,),start=NaT,end=NaT]  \n",
       "70:ee:50:7f:98:de  TS[shape=(17,),start=2021-08-13 22:00:00,end=2...  \n",
       "\n",
       "[748 rows x 6 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# station_ids = [\"70:ee:50:00:3b:86\", \"70:ee:50:00:53:26\"]\n",
    "station_ids = long_ts_df.index.get_level_values(\"id\").unique()\n",
    "long_ts_df = df.pivot_table(index=[\"id\", \"time\"], columns=\"variable\", values=\"value\")\n",
    "# TODO: TS objects of nan only should be nan values\n",
    "ts_df = pd.DataFrame(\n",
    "    {\n",
    "        variable: tstore.TSArray(\n",
    "            [\n",
    "                tstore.TS(long_ts_df.loc[station_id][variable].dropna())\n",
    "                for station_id in station_ids\n",
    "            ],\n",
    "        )\n",
    "        for variable in long_ts_df.columns\n",
    "    },\n",
    "    index=station_ids,\n",
    ")\n",
    "ts_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21",
   "metadata": {},
   "source": [
    "Note that we have many nan values. Finally, it is also possible to have a single `TS` object per module and station, in which case the `TS` objects of the first and third module would have multiple variables, all of which would be aligned in time:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "22",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>module_1</th>\n",
       "      <th>module_2</th>\n",
       "      <th>module_3</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>70:ee:50:00:10:f0</th>\n",
       "      <td>TS[shape=(247, 3),start=2021-08-12 13:00:00,en...</td>\n",
       "      <td>TS[shape=(247, 1),start=2021-08-12 13:00:00,en...</td>\n",
       "      <td>TS[shape=(247, 2),start=2021-08-12 13:00:00,en...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70:ee:50:00:17:8e</th>\n",
       "      <td>TS[shape=(41, 3),start=2021-08-17 00:00:00,end...</td>\n",
       "      <td>TS[shape=(41, 1),start=2021-08-17 00:00:00,end...</td>\n",
       "      <td>TS[shape=(41, 2),start=2021-08-17 00:00:00,end...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70:ee:50:00:20:88</th>\n",
       "      <td>TS[shape=(295, 3),start=2021-08-12 12:00:00,en...</td>\n",
       "      <td>TS[shape=(295, 1),start=2021-08-12 12:00:00,en...</td>\n",
       "      <td>TS[shape=(295, 2),start=2021-08-12 12:00:00,en...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70:ee:50:00:30:2a</th>\n",
       "      <td>TS[shape=(246, 3),start=2021-08-12 12:00:00,en...</td>\n",
       "      <td>TS[shape=(246, 1),start=2021-08-12 12:00:00,en...</td>\n",
       "      <td>TS[shape=(246, 2),start=2021-08-12 12:00:00,en...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70:ee:50:00:36:d0</th>\n",
       "      <td>TS[shape=(178, 3),start=2021-08-12 13:00:00,en...</td>\n",
       "      <td>TS[shape=(178, 1),start=2021-08-12 13:00:00,en...</td>\n",
       "      <td>TS[shape=(178, 2),start=2021-08-12 13:00:00,en...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70:ee:50:7a:c4:fc</th>\n",
       "      <td>TS[shape=(155, 3),start=2021-08-12 22:00:00,en...</td>\n",
       "      <td>TS[shape=(155, 1),start=2021-08-12 22:00:00,en...</td>\n",
       "      <td>TS[shape=(155, 2),start=2021-08-12 22:00:00,en...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70:ee:50:7a:d2:c2</th>\n",
       "      <td>TS[shape=(88, 3),start=2021-08-12 13:00:00,end...</td>\n",
       "      <td>TS[shape=(88, 1),start=2021-08-12 13:00:00,end...</td>\n",
       "      <td>TS[shape=(88, 2),start=2021-08-12 13:00:00,end...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70:ee:50:7a:dc:ce</th>\n",
       "      <td>TS[shape=(12, 3),start=2021-08-15 14:00:00,end...</td>\n",
       "      <td>TS[shape=(12, 1),start=2021-08-15 14:00:00,end...</td>\n",
       "      <td>TS[shape=(12, 2),start=2021-08-15 14:00:00,end...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70:ee:50:7a:e3:4a</th>\n",
       "      <td>TS[shape=(211, 3),start=2021-08-12 12:00:00,en...</td>\n",
       "      <td>TS[shape=(211, 1),start=2021-08-12 12:00:00,en...</td>\n",
       "      <td>TS[shape=(211, 2),start=2021-08-12 12:00:00,en...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70:ee:50:7f:98:de</th>\n",
       "      <td>TS[shape=(19, 3),start=2021-08-13 22:00:00,end...</td>\n",
       "      <td>TS[shape=(19, 1),start=2021-08-13 22:00:00,end...</td>\n",
       "      <td>TS[shape=(19, 2),start=2021-08-13 22:00:00,end...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>748 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                            module_1  \\\n",
       "id                                                                     \n",
       "70:ee:50:00:10:f0  TS[shape=(247, 3),start=2021-08-12 13:00:00,en...   \n",
       "70:ee:50:00:17:8e  TS[shape=(41, 3),start=2021-08-17 00:00:00,end...   \n",
       "70:ee:50:00:20:88  TS[shape=(295, 3),start=2021-08-12 12:00:00,en...   \n",
       "70:ee:50:00:30:2a  TS[shape=(246, 3),start=2021-08-12 12:00:00,en...   \n",
       "70:ee:50:00:36:d0  TS[shape=(178, 3),start=2021-08-12 13:00:00,en...   \n",
       "...                                                              ...   \n",
       "70:ee:50:7a:c4:fc  TS[shape=(155, 3),start=2021-08-12 22:00:00,en...   \n",
       "70:ee:50:7a:d2:c2  TS[shape=(88, 3),start=2021-08-12 13:00:00,end...   \n",
       "70:ee:50:7a:dc:ce  TS[shape=(12, 3),start=2021-08-15 14:00:00,end...   \n",
       "70:ee:50:7a:e3:4a  TS[shape=(211, 3),start=2021-08-12 12:00:00,en...   \n",
       "70:ee:50:7f:98:de  TS[shape=(19, 3),start=2021-08-13 22:00:00,end...   \n",
       "\n",
       "                                                            module_2  \\\n",
       "id                                                                     \n",
       "70:ee:50:00:10:f0  TS[shape=(247, 1),start=2021-08-12 13:00:00,en...   \n",
       "70:ee:50:00:17:8e  TS[shape=(41, 1),start=2021-08-17 00:00:00,end...   \n",
       "70:ee:50:00:20:88  TS[shape=(295, 1),start=2021-08-12 12:00:00,en...   \n",
       "70:ee:50:00:30:2a  TS[shape=(246, 1),start=2021-08-12 12:00:00,en...   \n",
       "70:ee:50:00:36:d0  TS[shape=(178, 1),start=2021-08-12 13:00:00,en...   \n",
       "...                                                              ...   \n",
       "70:ee:50:7a:c4:fc  TS[shape=(155, 1),start=2021-08-12 22:00:00,en...   \n",
       "70:ee:50:7a:d2:c2  TS[shape=(88, 1),start=2021-08-12 13:00:00,end...   \n",
       "70:ee:50:7a:dc:ce  TS[shape=(12, 1),start=2021-08-15 14:00:00,end...   \n",
       "70:ee:50:7a:e3:4a  TS[shape=(211, 1),start=2021-08-12 12:00:00,en...   \n",
       "70:ee:50:7f:98:de  TS[shape=(19, 1),start=2021-08-13 22:00:00,end...   \n",
       "\n",
       "                                                            module_3  \n",
       "id                                                                    \n",
       "70:ee:50:00:10:f0  TS[shape=(247, 2),start=2021-08-12 13:00:00,en...  \n",
       "70:ee:50:00:17:8e  TS[shape=(41, 2),start=2021-08-17 00:00:00,end...  \n",
       "70:ee:50:00:20:88  TS[shape=(295, 2),start=2021-08-12 12:00:00,en...  \n",
       "70:ee:50:00:30:2a  TS[shape=(246, 2),start=2021-08-12 12:00:00,en...  \n",
       "70:ee:50:00:36:d0  TS[shape=(178, 2),start=2021-08-12 13:00:00,en...  \n",
       "...                                                              ...  \n",
       "70:ee:50:7a:c4:fc  TS[shape=(155, 2),start=2021-08-12 22:00:00,en...  \n",
       "70:ee:50:7a:d2:c2  TS[shape=(88, 2),start=2021-08-12 13:00:00,end...  \n",
       "70:ee:50:7a:dc:ce  TS[shape=(12, 2),start=2021-08-15 14:00:00,end...  \n",
       "70:ee:50:7a:e3:4a  TS[shape=(211, 2),start=2021-08-12 12:00:00,en...  \n",
       "70:ee:50:7f:98:de  TS[shape=(19, 2),start=2021-08-13 22:00:00,end...  \n",
       "\n",
       "[748 rows x 3 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "module_dict = {\n",
    "    \"module_1\": [\"temperature\", \"humidity\", \"pressure\"],\n",
    "    \"module_2\": [\"rain_live\"],\n",
    "    \"module_3\": [\"wind_angle\", \"wind_strength\"],\n",
    "}\n",
    "ts_df = pd.DataFrame(\n",
    "    {\n",
    "        module: tstore.TSArray(\n",
    "            [\n",
    "                tstore.TS(long_ts_df.loc[station_id][variables])\n",
    "                for station_id in station_ids\n",
    "            ],\n",
    "        )\n",
    "        for module, variables in module_dict.items()\n",
    "    },\n",
    "    index=station_ids,\n",
    ")\n",
    "ts_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "23",
   "metadata": {},
   "source": [
    "TODO: geospatial example\n",
    "\n",
    "<a id='data-forms'></a>\n",
    "## Data forms\n",
    "\n",
    "Building upon [the documentation of seaborn](https://seaborn.pydata.org/tutorial/data_structure.html), we can define the following data forms:\n",
    "- **Long form**, where each variable is a column and each observation is a row.\n",
    "- **Wide form**, where columns and rows contain levels of a variable. Each observation is defined by its value at the cell and its coordinates in both the row and column indices.\n",
    "\n",
    "However, eventhough our dataset clearly looks more similar to the long form, it is not exactly a long form dataset since we do not have each variable as a column but instead have multiple variables in the same column. This kind of dataset can be colled \"messy\" or \"untidy\" data (see [the work on \"data tyding\"](https://vita.had.co.nz/papers/tidy-data.pdf) by Hadley Wickham), where each row corresponds to the unit of data collection (here, the meteorological measurement). As shown above, we can convert this dataset to a long form using [`pandas.DataFrame.pivot_table`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.pivot_table.html)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>variable</th>\n",
       "      <th>id</th>\n",
       "      <th>time</th>\n",
       "      <th>humidity</th>\n",
       "      <th>pressure</th>\n",
       "      <th>rain_live</th>\n",
       "      <th>temperature</th>\n",
       "      <th>wind_angle</th>\n",
       "      <th>wind_strength</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>70:ee:50:00:10:f0</td>\n",
       "      <td>2021-08-12 13:00:00</td>\n",
       "      <td>50.0</td>\n",
       "      <td>1023.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>28.1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>70:ee:50:00:10:f0</td>\n",
       "      <td>2021-08-12 14:00:00</td>\n",
       "      <td>49.0</td>\n",
       "      <td>1023.3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>29.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>70:ee:50:00:10:f0</td>\n",
       "      <td>2021-08-12 15:00:00</td>\n",
       "      <td>50.0</td>\n",
       "      <td>1023.3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>29.4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>70:ee:50:00:10:f0</td>\n",
       "      <td>2021-08-12 16:00:00</td>\n",
       "      <td>49.0</td>\n",
       "      <td>1022.2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>29.2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>70:ee:50:00:10:f0</td>\n",
       "      <td>2021-08-12 17:00:00</td>\n",
       "      <td>53.0</td>\n",
       "      <td>1023.3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>27.9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101454</th>\n",
       "      <td>70:ee:50:7f:98:de</td>\n",
       "      <td>2021-08-23 22:00:00</td>\n",
       "      <td>64.0</td>\n",
       "      <td>1019.8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101455</th>\n",
       "      <td>70:ee:50:7f:98:de</td>\n",
       "      <td>2021-08-25 01:00:00</td>\n",
       "      <td>69.0</td>\n",
       "      <td>1018.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17.5</td>\n",
       "      <td>52.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101456</th>\n",
       "      <td>70:ee:50:7f:98:de</td>\n",
       "      <td>2021-08-25 02:00:00</td>\n",
       "      <td>70.0</td>\n",
       "      <td>1018.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17.2</td>\n",
       "      <td>60.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101457</th>\n",
       "      <td>70:ee:50:7f:98:de</td>\n",
       "      <td>2021-08-25 03:00:00</td>\n",
       "      <td>71.0</td>\n",
       "      <td>1018.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>16.8</td>\n",
       "      <td>54.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101458</th>\n",
       "      <td>70:ee:50:7f:98:de</td>\n",
       "      <td>2021-08-25 04:00:00</td>\n",
       "      <td>73.0</td>\n",
       "      <td>1017.8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>16.7</td>\n",
       "      <td>50.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>101459 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "variable                 id                time  humidity  pressure  \\\n",
       "0         70:ee:50:00:10:f0 2021-08-12 13:00:00      50.0    1023.7   \n",
       "1         70:ee:50:00:10:f0 2021-08-12 14:00:00      49.0    1023.3   \n",
       "2         70:ee:50:00:10:f0 2021-08-12 15:00:00      50.0    1023.3   \n",
       "3         70:ee:50:00:10:f0 2021-08-12 16:00:00      49.0    1022.2   \n",
       "4         70:ee:50:00:10:f0 2021-08-12 17:00:00      53.0    1023.3   \n",
       "...                     ...                 ...       ...       ...   \n",
       "101454    70:ee:50:7f:98:de 2021-08-23 22:00:00      64.0    1019.8   \n",
       "101455    70:ee:50:7f:98:de 2021-08-25 01:00:00      69.0    1018.3   \n",
       "101456    70:ee:50:7f:98:de 2021-08-25 02:00:00      70.0    1018.1   \n",
       "101457    70:ee:50:7f:98:de 2021-08-25 03:00:00      71.0    1018.2   \n",
       "101458    70:ee:50:7f:98:de 2021-08-25 04:00:00      73.0    1017.8   \n",
       "\n",
       "variable  rain_live  temperature  wind_angle  wind_strength  \n",
       "0               NaN         28.1         NaN            NaN  \n",
       "1               NaN         29.5         NaN            NaN  \n",
       "2               NaN         29.4         NaN            NaN  \n",
       "3               NaN         29.2         NaN            NaN  \n",
       "4               NaN         27.9         NaN            NaN  \n",
       "...             ...          ...         ...            ...  \n",
       "101454          0.0         19.0        52.0            2.0  \n",
       "101455          0.0         17.5        52.0            3.0  \n",
       "101456          0.0         17.2        60.0            3.0  \n",
       "101457          0.0         16.8        54.0            2.0  \n",
       "101458          0.0         16.7        50.0            3.0  \n",
       "\n",
       "[101459 rows x 8 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "long_ts_df = df.pivot_table(\n",
    "    index=[\"id\", \"time\"],\n",
    "    columns=\"variable\",\n",
    "    values=\"value\",\n",
    ").reset_index()\n",
    "long_ts_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25",
   "metadata": {
    "lines_to_next_cell": 2
   },
   "source": [
    "## TStore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/tmp/dummy_tstore/tstore_metadata.yml\n",
      "/tmp/dummy_tstore/_attributes.parquet\n",
      "/tmp/dummy_tstore/70:ee:50:2c:95:e4/temperature/_common_metadata\n",
      "/tmp/dummy_tstore/70:ee:50:2c:95:e4/temperature/_metadata\n",
      "/tmp/dummy_tstore/70:ee:50:2c:95:e4/temperature/day=21/part-0.parquet\n",
      "/tmp/dummy_tstore/70:ee:50:2c:95:e4/temperature/day=15/part-0.parquet\n",
      "/tmp/dummy_tstore/70:ee:50:2c:95:e4/temperature/day=13/part-0.parquet\n",
      "/tmp/dummy_tstore/70:ee:50:2c:95:e4/temperature/day=18/part-0.parquet\n",
      "/tmp/dummy_tstore/70:ee:50:2c:95:e4/temperature/day=24/part-0.parquet\n",
      "/tmp/dummy_tstore/70:ee:50:2c:95:e4/temperature/day=19/part-0.parquet\n",
      "...\n",
      "/tmp/dummy_tstore/70:ee:50:3c:28:cc/humidity/day=18/part-0.parquet\n",
      "/tmp/dummy_tstore/70:ee:50:3c:28:cc/humidity/day=24/part-0.parquet\n",
      "/tmp/dummy_tstore/70:ee:50:3c:28:cc/humidity/day=20/part-0.parquet\n",
      "/tmp/dummy_tstore/70:ee:50:3c:28:cc/humidity/day=19/part-0.parquet\n",
      "/tmp/dummy_tstore/70:ee:50:3c:28:cc/humidity/day=22/part-0.parquet\n",
      "/tmp/dummy_tstore/70:ee:50:3c:28:cc/humidity/day=16/part-0.parquet\n",
      "/tmp/dummy_tstore/70:ee:50:3c:28:cc/humidity/day=23/part-0.parquet\n",
      "/tmp/dummy_tstore/70:ee:50:3c:28:cc/humidity/day=14/part-0.parquet\n",
      "/tmp/dummy_tstore/70:ee:50:3c:28:cc/humidity/day=12/part-0.parquet\n",
      "/tmp/dummy_tstore/70:ee:50:3c:28:cc/humidity/day=17/part-0.parquet\n",
      "`to_tstore` time (s): 6.651559114456177\n",
      "`open_tslong` time (s): 3.2708444595336914\n"
     ]
    }
   ],
   "source": [
    "def tstore_tree(base_dir, head=10, tail=10):\n",
    "    result = [\n",
    "        path.join(dp, f) for dp, dn, filenames in os.walk(base_dir) for f in filenames\n",
    "    ]\n",
    "    return result[:head] + [\"...\"] + result[-tail:]\n",
    "\n",
    "\n",
    "# try round-trip\n",
    "# base_dir = \"/tmp/dummy_tstore\"\n",
    "tmp_dir = \"/tmp/dummy_tstore\"\n",
    "tstore_structure = \"id-var\"\n",
    "overwrite = True\n",
    "id_var = \"id\"\n",
    "time_var = \"time\"\n",
    "# static_variables = [\"variable\"]\n",
    "geometry = None  # NOT IMPLEMENTED YET\n",
    "\n",
    "# Same partitioning for all TS\n",
    "partitioning = \"day\"\n",
    "\n",
    "# Each timeseries is a TS object\n",
    "# TODO: how do I put humidity/temperature in the same TS object?\n",
    "# the line below does not work\n",
    "ts_variables = [\"humidity\", \"temperature\"]\n",
    "# ts_variables = {var: [\"id\", \"value\"] for var in ts_df[\"variable\"].unique()}\n",
    "# Group multiple timeseries into one TS object\n",
    "# ts_variables = {\"precipitation\": [\"name\", \"id\", \"x\", \"y\"]}\n",
    "\n",
    "with tempfile.TemporaryDirectory() as _tmp_dir:\n",
    "    start_write = time.time()\n",
    "    to_tstore(\n",
    "        long_ts_df,\n",
    "        # TSTORE options\n",
    "        tmp_dir,\n",
    "        # DFLONG attributes\n",
    "        id_var=id_var,\n",
    "        time_var=time_var,\n",
    "        ts_variables=ts_variables,\n",
    "        # static_variables=static_variables,\n",
    "        # TSTORE options\n",
    "        partitioning=partitioning,\n",
    "        tstore_structure=tstore_structure,\n",
    "        overwrite=overwrite,\n",
    "    )\n",
    "    end_write = time.time()\n",
    "    for line in tstore_tree(tmp_dir):\n",
    "        print(line)\n",
    "    print(\"`to_tstore` time (s):\", end_write - start_write)\n",
    "\n",
    "    start_read = time.time()\n",
    "    df = open_tslong(tmp_dir, ts_variables=list(ts_variables))\n",
    "    print(\"`open_tslong` time (s):\", time.time() - start_read)"
   ]
  }
 ],
 "metadata": {
  "jupytext": {
   "cell_metadata_filter": "tags,-all"
  },
  "kernelspec": {
   "display_name": "Python (tstore)",
   "language": "python",
   "name": "tstore"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}