merge-cues.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Merge CUES observations into a single file\n",
    "\n",
    "From directories of CUES radiation and temperature level 1 csv files, align and merge while doing some data cleanup. Save out as a pandas dataframe to a pickle file.\n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import datetime as dt\n",
    "import xarray as xr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getListOfFiles(dirName):\n",
    "    # create a list of file and sub directories \n",
    "    # names in the given directory \n",
    "    # https://thispointer.com/python-how-to-get-list-of-files-in-directory-and-sub-directories/\n",
    "    listOfFile = os.listdir(dirName)\n",
    "    allFiles = list()\n",
    "    # Iterate over all the entries\n",
    "    for entry in listOfFile:\n",
    "        # Create full path\n",
    "        fullPath = os.path.join(dirName, entry)\n",
    "        # If entry is a directory then get the list of files in this directory \n",
    "        if os.path.isdir(fullPath):\n",
    "            allFiles = allFiles + getListOfFiles(fullPath)\n",
    "        else:\n",
    "            allFiles.append(fullPath)\n",
    "                \n",
    "    return allFiles \n",
    "\n",
    "def getCUESdata(dirName):\n",
    "    # Get all the files in this directory we'll want to open\n",
    "    cues_files = getListOfFiles(dirName)\n",
    "    \n",
    "    # Open all the files and concat together in a pandas dataframe\n",
    "    cues_data = []\n",
    "    _ = [cues_data.append(pd.read_csv(i)) for i in cues_files]\n",
    "    cues_data = pd.concat(cues_data)\n",
    "    \n",
    "    # Convert the 'DateTime' text strings to pandas datetime objects\n",
    "    cues_data['datetime']  = pd.to_datetime(cues_data['MeasDateTime'])\n",
    "    \n",
    "    #Convert pandas dataframe to xarray dataset and and make our local time datetimes the index\n",
    "    _cues_data = cues_data.set_index('datetime')\n",
    "    _cues_data.sort_index(inplace=True)\n",
    "    cues_ds = _cues_data.to_xarray()\n",
    "    \n",
    "    # drop old time field\n",
    "    cues_ds = cues_ds.drop_vars(names='MeasDateTime')\n",
    "    \n",
    "    return cues_ds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "## Open data files\n",
    "\n",
    "Specify directories containing CUES Level 1 csv files for radiation and temperature data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_directory = r'data\\temperature'\n",
    "rad_directory = r'data\\radiation'\n",
    "wind_directory = r'data\\wind'\n",
    "precip_directory = r'data\\precipitation'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Open the CUES data files, and concatenate them together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# temperature\n",
    "cues_temp = getCUESdata(temp_directory)\n",
    "# select only the unique datetime indices we have now (But why are there duplicates and where are they?)\n",
    "_, index = np.unique(cues_temp['datetime'], return_index=True)\n",
    "cues_temp = cues_temp.isel(datetime=index)\n",
    "\n",
    "# radiation\n",
    "cues_rad = getCUESdata(rad_directory)\n",
    "\n",
    "# wind\n",
    "cues_wind = getCUESdata(wind_directory)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Open the 3D wind vector data from the sonic anemometer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "cues_3d_wind = pd.read_csv(r'data\\CUESWinds201703through201704.csv')\n",
    "# Convert the 'DateTime' text strings to pandas datetime objects\n",
    "cues_3d_wind['datetime']  = pd.to_datetime(cues_3d_wind['TIMESTAMP'])\n",
    "#Convert pandas dataframe to xarray dataset and and make our local time datetimes the index\n",
    "_cues_3d_wind = cues_3d_wind.set_index('datetime')\n",
    "_cues_3d_wind.sort_index(inplace=True)\n",
    "cues_3d_wind = _cues_3d_wind.to_xarray()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Merge the rad and temp datasets together, then save out to a file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "cues_ds = xr.merge([cues_temp, cues_rad, cues_wind, cues_3d_wind])\n",
    "# Convert back to pandas dataframe, save out to pickle file\n",
    "cues_ds.to_dataframe().to_pickle('data/CUES_L1_Temp_Rad_Wind_Mar-Apr2017.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n",
    "Open the CUES precip data (these are daily values, I won't merge with the rest of the data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "cues_precip = getCUESdata(precip_directory)\n",
    "cues_precip.to_dataframe().to_pickle('data/CUES_L1_Precip_Mar-Apr2017.pkl')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}