Correction_for train data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YgPItvgqpbfr"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "#%matplotlib inline\n",
    "import time\n",
    "from datetime import datetime\n",
    "import rpy2.robjects as robjects\n",
    "from rpy2.robjects.packages import importr\n",
    "from rpy2.robjects import r, numpy2ri, FloatVector, pandas2ri\n",
    "from datetime import timedelta\n",
    "from IPython import get_ipython\n",
    "import sys"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "WjFsgdvQbpXy"
   },
   "source": [
    "# Observation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "0-cjtFj-bpXz"
   },
   "source": [
    "## 定義所採納的時間"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 198
    },
    "colab_type": "code",
    "id": "l0mA1mB4ni6N",
    "outputId": "2eb02d47-d57d-4435-f0ac-cbc697bc7571"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nstart = str(sys.argv[1]) + \\' 00:00:00\\' #input(\\'起始日期：(格式:2019-01-01)\\')\\nend = str(sys.argv[2]) + \\' 23:00:00\\' #input(\\'結束日期：(格式:2019-10-17)\\')\\ntime_step = int(sys.argv[3]) #int(input(\\'預報小時：(格式:96)\\'))\\nver = str(sys.argv[4]).split(\",\") #[int(num) for num in input(\"請輸入全部的版本(格式：17 18),以空白鍵區隔：\").split(\",\")] \\n'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "start = str(sys.argv[1]) + ' 00:00:00' #input('起始日期：(格式:2019-01-01)')\n",
    "end = str(sys.argv[2]) + ' 23:00:00' #input('結束日期：(格式:2019-10-17)')\n",
    "time_step = int(sys.argv[3]) #int(input('預報小時：(格式:96)'))\n",
    "ver = str(sys.argv[4]).split(\",\") #[int(num) for num in input(\"請輸入全部的版本(格式：17 18),以空白鍵區隔：\").split(\",\")] \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jdigckMZLQXy"
   },
   "outputs": [],
   "source": [
    "def data_cleaning(df):\n",
    "    \n",
    "    df['year'] = df['year'].astype(str)\n",
    "    df['month'] = df['month'].astype(str)\n",
    "    df['day'] = df['day'].astype(str)\n",
    "    df['hour'] = df['hour'].astype(str)\n",
    "    df['dateTime'] = df['year'] + '-' + df['month'].str.zfill(2) + '-' + df['day'].str.zfill(2) + ' ' + df['hour'].str.zfill(2) +':'+ '00:00 '\n",
    "    df['dateTime'] = pd.to_datetime(df['dateTime']).apply(lambda x:int(time.mktime(x.timetuple())))\n",
    "    df.sort_values(by=['dateTime', 'stationID'], inplace = True)\n",
    "\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gzqYWVYQpbf6"
   },
   "outputs": [],
   "source": [
    "time_interval = time_step // 24\n",
    "start = time.strptime(start, \"%Y-%m-%d %H:%M:%S\")\n",
    "end = time.strptime(end, \"%Y-%m-%d %H:%M:%S\")\n",
    "\n",
    "# 轉換為時間戳:\n",
    "start = int(time.mktime(start)) - 3600*24\n",
    "end_pre = int(time.mktime(end))\n",
    "en_ob = int(time.mktime(end)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "讀取新檔\n"
     ]
    }
   ],
   "source": [
    "new_observation = pd.read_table(\"opendata_epa.txt\") \n",
    "new_observation = data_cleaning(new_observation)\n",
    "file_name = 'corr_old_opendata_epa_' + str(time_step) + '.txt'\n",
    "try:\n",
    "    old_observation = pd.read_table(file_name)\n",
    "    old_observation = data_cleaning(old_observation)\n",
    "    old_end = old_observation['dateTime'].iloc[-1] + 3600\n",
    "    new_observation = new_observation[new_observation['dateTime']>=old_end]\n",
    "    df_observation = pd.concat([old_observation, new_observation], axis = 0, ignore_index=True)\n",
    "    print('舊檔新檔合併')\n",
    "except IOError:\n",
    "    df_observation = new_observation\n",
    "    print('讀取新檔')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 195
    },
    "colab_type": "code",
    "id": "hCcmpr-ypbf8",
    "outputId": "4ccf55ee-a6eb-4627-8bac-09aec92c006a"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\User\\Anaconda3\\envs\\tf2_rpy2\\lib\\site-packages\\ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n",
      "C:\\Users\\User\\Anaconda3\\envs\\tf2_rpy2\\lib\\site-packages\\ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>hour</th>\n",
       "      <th>stationID</th>\n",
       "      <th>PMf</th>\n",
       "      <th>dateTime</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>553571</td>\n",
       "      <td>2019</td>\n",
       "      <td>10</td>\n",
       "      <td>31</td>\n",
       "      <td>23</td>\n",
       "      <td>EPA084</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-10-31 23:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>553572</td>\n",
       "      <td>2019</td>\n",
       "      <td>10</td>\n",
       "      <td>31</td>\n",
       "      <td>23</td>\n",
       "      <td>EPA101</td>\n",
       "      <td>22.0</td>\n",
       "      <td>2019-10-31 23:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>553573</td>\n",
       "      <td>2019</td>\n",
       "      <td>10</td>\n",
       "      <td>31</td>\n",
       "      <td>23</td>\n",
       "      <td>EPA102</td>\n",
       "      <td>19.0</td>\n",
       "      <td>2019-10-31 23:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>553574</td>\n",
       "      <td>2019</td>\n",
       "      <td>10</td>\n",
       "      <td>31</td>\n",
       "      <td>23</td>\n",
       "      <td>EPA104</td>\n",
       "      <td>22.0</td>\n",
       "      <td>2019-10-31 23:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>553575</td>\n",
       "      <td>2019</td>\n",
       "      <td>10</td>\n",
       "      <td>31</td>\n",
       "      <td>23</td>\n",
       "      <td>EPA110</td>\n",
       "      <td>27.0</td>\n",
       "      <td>2019-10-31 23:00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        year month day hour stationID   PMf             dateTime\n",
       "553571  2019    10  31   23    EPA084   NaN  2019-10-31 23:00:00\n",
       "553572  2019    10  31   23    EPA101  22.0  2019-10-31 23:00:00\n",
       "553573  2019    10  31   23    EPA102  19.0  2019-10-31 23:00:00\n",
       "553574  2019    10  31   23    EPA104  22.0  2019-10-31 23:00:00\n",
       "553575  2019    10  31   23    EPA110  27.0  2019-10-31 23:00:00"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_observation = df_observation[df_observation['dateTime'].between(start - 3600 * 24, en_ob + (time_interval) * 3600 * 24)]\n",
    "df_observation['dateTime'] = df_observation['dateTime'].apply(lambda x:time.strftime(\"%Y-%m-%d %H:%M:%S\",time.localtime(x)))\n",
    "df_observation.sort_values(by=['dateTime', 'stationID'], inplace = True)\n",
    "opendata_station = list(pd.unique(df_observation['stationID']))\n",
    "\n",
    "df_observation.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Ua1LJ85fpbf_"
   },
   "source": [
    "## 移除測站'EPA101',' EPA084',' EPA104'(因為全部皆為NAN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 195
    },
    "colab_type": "code",
    "id": "aO81QQRjpbgA",
    "outputId": "5ac87b33-edf8-412f-f1aa-1006025a9b3e"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>hour</th>\n",
       "      <th>stationID</th>\n",
       "      <th>PMf</th>\n",
       "      <th>dateTime</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>492592</td>\n",
       "      <td>2019</td>\n",
       "      <td>9</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>EPA001</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2019-09-29 00:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>492593</td>\n",
       "      <td>2019</td>\n",
       "      <td>9</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>EPA002</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2019-09-29 00:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>492594</td>\n",
       "      <td>2019</td>\n",
       "      <td>9</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>EPA003</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2019-09-29 00:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>492595</td>\n",
       "      <td>2019</td>\n",
       "      <td>9</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>EPA004</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-09-29 00:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>492596</td>\n",
       "      <td>2019</td>\n",
       "      <td>9</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>EPA005</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2019-09-29 00:00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        year month day hour stationID  PMf             dateTime\n",
       "492592  2019     9  29    0    EPA001  4.0  2019-09-29 00:00:00\n",
       "492593  2019     9  29    0    EPA002  4.0  2019-09-29 00:00:00\n",
       "492594  2019     9  29    0    EPA003  4.0  2019-09-29 00:00:00\n",
       "492595  2019     9  29    0    EPA004  1.0  2019-09-29 00:00:00\n",
       "492596  2019     9  29    0    EPA005  5.0  2019-09-29 00:00:00"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#pd.value_counts(df['stationID'])\n",
    "df_observation = df_observation[df_observation['stationID']!='EPA101']\n",
    "df_observation = df_observation[df_observation['stationID']!='EPA084']\n",
    "df_observation = df_observation[df_observation['stationID']!='EPA104']\n",
    "\n",
    "df_observation.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nC_PFCyHbpYD"
   },
   "outputs": [],
   "source": [
    "station_num = len(np.unique(df_observation['stationID'])) ##測站總數"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NU912CU1pbgB"
   },
   "source": [
    "## 進行時間缺漏值填補"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-UiSYrZwpbgD"
   },
   "outputs": [],
   "source": [
    "stations = list(pd.unique(df_observation['stationID']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "5dEZLEaapbgE",
    "outputId": "5b8bdce4-8b69-4882-b772-754d53e41baa"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------observation 補缺失日期開始------\n"
     ]
    }
   ],
   "source": [
    "print('------observation 補缺失日期開始------')\n",
    "df_observation['dateTime'] = pd.to_datetime(df_observation['dateTime']).apply(lambda x:int(time.mktime(x.timetuple())))\n",
    "df_observation.index = np.arange(len(df_observation))\n",
    "a = df_observation['dateTime'][0]\n",
    "b = int(df_observation['dateTime'][-1:].values)\n",
    "all_date = list(range(a,b,3600))+[b]\n",
    "df_add = df_observation[: 1].copy()\n",
    "for name in stations:\n",
    "    #print('name', name)\n",
    "    unnormal = list(df_observation[df_observation['stationID'] == name]['dateTime'])\n",
    "    dates = list(set(all_date).difference(set(unnormal)))\n",
    "    #print('date',len(dates))\n",
    "    \n",
    "    for date in dates:\n",
    "        df_add['dateTime'] = date\n",
    "        df_add['stationID'] = name\n",
    "        df_add['PMf'] = np.nan\n",
    "        df_observation = pd.concat([df_observation, df_add])\n",
    "    df_observation.sort_values(by=['dateTime','stationID'], inplace = True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 195
    },
    "colab_type": "code",
    "id": "c0D3OrcnpbgG",
    "outputId": "f1c550e8-c9ca-4c97-93de-c4d9d2b63604"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>hour</th>\n",
       "      <th>stationID</th>\n",
       "      <th>PMf</th>\n",
       "      <th>dateTime</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>58603</td>\n",
       "      <td>2019</td>\n",
       "      <td>10</td>\n",
       "      <td>31</td>\n",
       "      <td>23</td>\n",
       "      <td>EPA071</td>\n",
       "      <td>37.0</td>\n",
       "      <td>2019-10-31 23:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>58604</td>\n",
       "      <td>2019</td>\n",
       "      <td>10</td>\n",
       "      <td>31</td>\n",
       "      <td>23</td>\n",
       "      <td>EPA072</td>\n",
       "      <td>43.0</td>\n",
       "      <td>2019-10-31 23:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>58605</td>\n",
       "      <td>2019</td>\n",
       "      <td>10</td>\n",
       "      <td>31</td>\n",
       "      <td>23</td>\n",
       "      <td>EPA080</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2019-10-31 23:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>58606</td>\n",
       "      <td>2019</td>\n",
       "      <td>10</td>\n",
       "      <td>31</td>\n",
       "      <td>23</td>\n",
       "      <td>EPA102</td>\n",
       "      <td>19.0</td>\n",
       "      <td>2019-10-31 23:00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>58607</td>\n",
       "      <td>2019</td>\n",
       "      <td>10</td>\n",
       "      <td>31</td>\n",
       "      <td>23</td>\n",
       "      <td>EPA110</td>\n",
       "      <td>27.0</td>\n",
       "      <td>2019-10-31 23:00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       year  month  day  hour stationID   PMf             dateTime\n",
       "58603  2019     10   31    23    EPA071  37.0  2019-10-31 23:00:00\n",
       "58604  2019     10   31    23    EPA072  43.0  2019-10-31 23:00:00\n",
       "58605  2019     10   31    23    EPA080   4.0  2019-10-31 23:00:00\n",
       "58606  2019     10   31    23    EPA102  19.0  2019-10-31 23:00:00\n",
       "58607  2019     10   31    23    EPA110  27.0  2019-10-31 23:00:00"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " df_observation['dateTime'] = df_observation['dateTime'].apply(lambda x:time.strftime(\"%Y-%m-%d %H:%M:%S\",time.localtime(x)))\n",
    "df_observation['month'] = pd.to_datetime(df_observation['dateTime']).dt.month\n",
    "df_observation['year'] = pd.to_datetime(df_observation['dateTime']).dt.year\n",
    "df_observation['day'] = pd.to_datetime(df_observation['dateTime']).dt.day\n",
    "df_observation['hour'] = pd.to_datetime(df_observation['dateTime']).dt.hour\n",
    "\n",
    "df_observation.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "161URL2BpbgH"
   },
   "source": [
    "### 檢查補植後結果 1:補值成功 ; 不為1:補值失敗"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "qSoK8PjXpbgI",
    "outputId": "e3ee44d6-f615-4e33-94f1-3a90eb8fffc7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "observation補缺失日期成功\n"
     ]
    }
   ],
   "source": [
    "if len(np.unique(pd.value_counts(df_observation['stationID'])))==1:\n",
    "    print(\"observation補缺失日期成功\")\n",
    "else:\n",
    "    print(\"observation補缺失日期失敗\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nnyhlW57IWdA"
   },
   "outputs": [],
   "source": [
    "df_observation.to_csv(file_name, sep='\\t', index=False)  ##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gUMBGkmMPJ2e"
   },
   "outputs": [],
   "source": [
    "df_observation['date'] = df_observation['dateTime'].apply(lambda x:time.strftime(\"%Y-%m-%d\",time.strptime(x,'%Y-%m-%d %H:%M:%S')))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "wyCYtTTbpbgK"
   },
   "source": [
    "## 轉為一天為單位的矩陣"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "keKt7ZRhpbgO",
    "outputId": "f6337bcb-3975-4a1c-e5ca-a49407d4cce8"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 4.,  4.,  4., ...,  3., 30., 11.],\n",
       "       [ 8.,  8.,  7., ...,  7.,  9., 13.],\n",
       "       [ 6.,  5.,  8., ...,  3., 39., 23.],\n",
       "       ...,\n",
       "       [ 6.,  6.,  3., ...,  0., 61., 18.],\n",
       "       [19., 19., 24., ...,  7., 31., 24.],\n",
       "       [27., 26., 40., ...,  4., 19., 27.]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "one_day_df = np.array(df_observation['PMf']).reshape(-1, station_num * 24)\n",
    "one_day_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>1766</th>\n",
       "      <th>1767</th>\n",
       "      <th>1768</th>\n",
       "      <th>1769</th>\n",
       "      <th>1770</th>\n",
       "      <th>1771</th>\n",
       "      <th>1772</th>\n",
       "      <th>1773</th>\n",
       "      <th>1774</th>\n",
       "      <th>1775</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2019-10-27</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>13.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-28</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-29</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>...</td>\n",
       "      <td>17.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-30</td>\n",
       "      <td>19.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>...</td>\n",
       "      <td>12.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-31</td>\n",
       "      <td>27.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>...</td>\n",
       "      <td>25.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>27.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1776 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            0     1     2     3     4     5     6     7     8     9     ...  \\\n",
       "2019-10-27  15.0  20.0  14.0  15.0  15.0  19.0  15.0  20.0  24.0  20.0  ...   \n",
       "2019-10-28   8.0   9.0   8.0   5.0  11.0  10.0  10.0  14.0  15.0  12.0  ...   \n",
       "2019-10-29   6.0   6.0   3.0   3.0   4.0   2.0   5.0   8.0   5.0   6.0  ...   \n",
       "2019-10-30  19.0  19.0  24.0  15.0  18.0   7.0  12.0  12.0  21.0  18.0  ...   \n",
       "2019-10-31  27.0  26.0  40.0  24.0  29.0  32.0  29.0  32.0  34.0  27.0  ...   \n",
       "\n",
       "            1766  1767  1768  1769  1770  1771  1772  1773  1774  1775  \n",
       "2019-10-27   4.0  26.0   9.0  23.0   8.0  27.0  33.0  13.0  10.0  13.0  \n",
       "2019-10-28   4.0  16.0   9.0  13.0   3.0  14.0  21.0  11.0   2.0   7.0  \n",
       "2019-10-29  17.0  25.0  17.0  21.0  19.0  25.0  25.0   0.0  61.0  18.0  \n",
       "2019-10-30  12.0  42.0  29.0  45.0  25.0  61.0  60.0   7.0  31.0  24.0  \n",
       "2019-10-31  25.0  29.0  21.0  32.0  20.0  37.0  43.0   4.0  19.0  27.0  \n",
       "\n",
       "[5 rows x 1776 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index = pd.unique(df_observation['date'] ).astype(str)\n",
    "observation_z = pd.DataFrame(columns=np.arange(station_num*24), data = one_day_df,index = index)\n",
    "observation_y1 =  observation_z.copy()\n",
    "observation_z.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kaj3rvESbpYR"
   },
   "source": [
    "## 將PMF=NAN進行補值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EyHfX7Q3bpYS"
   },
   "source": [
    "### 引入R的套件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PAC7ek6qbpYS"
   },
   "outputs": [],
   "source": [
    "#%load_ext rpy2.ipython"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "RcCV-1x-bpYU",
    "outputId": "97ad447e-3a92-4a02-e43e-688c6ed45bbd"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <span>BoolVector with 1 elements.</span>\n",
       "    <table>\n",
       "      <tbody>\n",
       "      <tr>\n",
       "      \n",
       "      <td>\n",
       "               0\n",
       "      </td>\n",
       "      \n",
       "      </tr>\n",
       "      </tbody>\n",
       "    </table>\n",
       "    "
      ],
      "text/plain": [
       "R object with classes: ('logical',) mapped to:\n",
       "<BoolVector - Python:0x000001C4D13DC648 / R:0x000001C4D24F6F40>\n",
       "[       0]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "robjects.r(\"install.packages('devtools')\")\n",
    "\n",
    "robjects.r('source(\"https://bioconductor.org/biocLite.R\")')\n",
    "r('''devtools::install_github(\"marchtaylor/sinkr\")''')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZBC1DS2PbpYX"
   },
   "outputs": [],
   "source": [
    "devtools = importr('devtools')\n",
    "sinkr = importr('sinkr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1UBn8hA2bpYZ",
    "outputId": "0de39ea4-2d4f-4e2c-b3d8-46f848d2cfd0"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <span>R/rpy2 DataFrame (33 x 1776)</span>\n",
       "    <table>\n",
       "      <thead>\n",
       "        <tr>\n",
       "        \n",
       "          <th>X0</th>\n",
       "        \n",
       "          <th>X1</th>\n",
       "        \n",
       "          <th>X2</th>\n",
       "        \n",
       "          <th>...</th>\n",
       "        \n",
       "          <th>X1773</th>\n",
       "        \n",
       "          <th>X1774</th>\n",
       "        \n",
       "          <th>X1775</th>\n",
       "        \n",
       "        </tr>\n",
       "      </thead>\n",
       "      <tbody>\n",
       "      \n",
       "      <tr>\n",
       "      \n",
       "      <td>\n",
       "        4.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        4.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        4.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        3.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        30.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        11.000000\n",
       "      </td>\n",
       "      \n",
       "      </tr>\n",
       "      \n",
       "      <tr>\n",
       "      \n",
       "      <td>\n",
       "        8.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        8.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        7.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        7.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        9.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        13.000000\n",
       "      </td>\n",
       "      \n",
       "      </tr>\n",
       "      \n",
       "      <tr>\n",
       "      \n",
       "      <td>\n",
       "        6.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        5.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        8.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        3.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        39.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        23.000000\n",
       "      </td>\n",
       "      \n",
       "      </tr>\n",
       "      \n",
       "      <tr>\n",
       "      \n",
       "      <td>\n",
       "        10.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        17.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        9.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        12.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        62.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        52.000000\n",
       "      </td>\n",
       "      \n",
       "      </tr>\n",
       "      \n",
       "      <tr>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      </tr>\n",
       "      \n",
       "      <tr>\n",
       "      \n",
       "      <td>\n",
       "        8.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        9.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        8.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        11.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        2.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        7.000000\n",
       "      </td>\n",
       "      \n",
       "      </tr>\n",
       "      \n",
       "      <tr>\n",
       "      \n",
       "      <td>\n",
       "        6.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        6.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        3.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        0.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        61.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        18.000000\n",
       "      </td>\n",
       "      \n",
       "      </tr>\n",
       "      \n",
       "      <tr>\n",
       "      \n",
       "      <td>\n",
       "        19.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        19.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        24.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        7.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        31.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        24.000000\n",
       "      </td>\n",
       "      \n",
       "      </tr>\n",
       "      \n",
       "      <tr>\n",
       "      \n",
       "      <td>\n",
       "        27.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        26.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        40.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        4.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        19.000000\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        27.000000\n",
       "      </td>\n",
       "      \n",
       "      </tr>\n",
       "      \n",
       "      </tbody>\n",
       "    </table>\n",
       "    "
      ],
      "text/plain": [
       "R object with classes: ('data.frame',) mapped to:\n",
       "<DataFrame - Python:0x000001C4DD85D0C8 / R:0x000001C4DE6F8570>\n",
       "[FloatVector, FloatVector, FloatVector, FloatVector, ..., FloatVector, FloatVector, FloatVector, FloatVector]\n",
       "  X0: <class 'rpy2.robjects.vectors.FloatVector'>\n",
       "  R object with classes: ('numeric',) mapped to:\n",
       "<FloatVector - Python:0x000001C4D176BE48 / R:0x000001C4CEBD7F70>\n",
       "[4.000000, 8.000000, 6.000000, 10.000000, ..., 8.000000, 6.000000, 19.000000, 27.000000]\n",
       "  X1: <class 'rpy2.robjects.vectors.FloatVector'>\n",
       "  R object with classes: ('numeric',) mapped to:\n",
       "<FloatVector - Python:0x000001C4DC24E1C8 / R:0x000001C4CE3926C0>\n",
       "[4.000000, 8.000000, 5.000000, 17.000000, ..., 9.000000, 6.000000, 19.000000, 26.000000]\n",
       "  X2: <class 'rpy2.robjects.vectors.FloatVector'>\n",
       "  R object with classes: ('numeric',) mapped to:\n",
       "<FloatVector - Python:0x000001C4DC24E388 / R:0x000001C4DB14CE68>\n",
       "[4.000000, 7.000000, 8.000000, 9.000000, ..., 8.000000, 3.000000, 24.000000, 40.000000]\n",
       "  X3: <class 'rpy2.robjects.vectors.FloatVector'>\n",
       "  R object with classes: ('numeric',) mapped to:\n",
       "<FloatVector - Python:0x000001C4DC24E488 / R:0x000001C4D8855758>\n",
       "[1.000000, 5.000000, 1.000000, 5.000000, ..., 5.000000, 3.000000, 15.000000, 24.000000]\n",
       "...\n",
       "  X5: <class 'rpy2.robjects.vectors.FloatVector'>\n",
       "  R object with classes: ('numeric',) mapped to:\n",
       "<FloatVector - Python:0x000001C4DC24E608 / R:0x000001C4DB57B4B8>\n",
       "[29.000000, 5.000000, 9.000000, 41.000000, ..., 21.000000, 25.000000, 60.000000, 43.000000]\n",
       "  X6: <class 'rpy2.robjects.vectors.FloatVector'>\n",
       "  R object with classes: ('numeric',) mapped to:\n",
       "<FloatVector - Python:0x000001C4DC24E708 / R:0x000001C4DB4E3978>\n",
       "[3.000000, 7.000000, 3.000000, 12.000000, ..., 11.000000, 0.000000, 7.000000, 4.000000]\n",
       "  X7: <class 'rpy2.robjects.vectors.FloatVector'>\n",
       "  R object with classes: ('numeric',) mapped to:\n",
       "<FloatVector - Python:0x000001C4DC24E7C8 / R:0x000001C4DB4E3CF0>\n",
       "[30.000000, 9.000000, 39.000000, 62.000000, ..., 2.000000, 61.000000, 31.000000, 19.000000]\n",
       "  X8: <class 'rpy2.robjects.vectors.FloatVector'>\n",
       "  R object with classes: ('numeric',) mapped to:\n",
       "<FloatVector - Python:0x000001C4DC24E8C8 / R:0x000001C4DB4E4128>\n",
       "[11.000000, 13.000000, 23.000000, 52.000000, ..., 7.000000, 18.000000, 24.000000, 27.000000]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "numpy2ri.activate()\n",
    "pandas2ri.activate()\n",
    "rpy2_observation_z = pandas2ri.py2ri(observation_z)\n",
    "robjects.r.assign('rpy2_observation_z', rpy2_observation_z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bJFVN1KabpYb",
    "outputId": "93aad563-35c4-4186-ac34-025b43054b37"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\User\\Anaconda3\\envs\\tf2_rpy2\\lib\\site-packages\\rpy2-2.9.4-py3.6-win-amd64.egg\\rpy2\\robjects\\pandas2ri.py:191: FutureWarning: from_items is deprecated. Please use DataFrame.from_dict(dict(items), ...) instead. DataFrame.from_dict(OrderedDict(items)) may be used to preserve the key order.\n",
      "  res = PandasDataFrame.from_items(items)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X0</th>\n",
       "      <th>X1</th>\n",
       "      <th>X2</th>\n",
       "      <th>X3</th>\n",
       "      <th>X4</th>\n",
       "      <th>X5</th>\n",
       "      <th>X6</th>\n",
       "      <th>X7</th>\n",
       "      <th>X8</th>\n",
       "      <th>X9</th>\n",
       "      <th>...</th>\n",
       "      <th>X1766</th>\n",
       "      <th>X1767</th>\n",
       "      <th>X1768</th>\n",
       "      <th>X1769</th>\n",
       "      <th>X1770</th>\n",
       "      <th>X1771</th>\n",
       "      <th>X1772</th>\n",
       "      <th>X1773</th>\n",
       "      <th>X1774</th>\n",
       "      <th>X1775</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>18.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>11.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>18.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>13.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>...</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>26.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>23.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>10.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>...</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>43.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>52.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>25.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>...</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>53.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>42.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>27.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>38.0</td>\n",
       "      <td>...</td>\n",
       "      <td>18.072161</td>\n",
       "      <td>26.0</td>\n",
       "      <td>36.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>43.372483</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6 rows × 1776 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     X0    X1    X2    X3    X4    X5    X6    X7    X8    X9  ...      X1766  \\\n",
       "0   4.0   4.0   4.0   1.0   5.0   1.0   6.0   6.0   8.0   5.0  ...   8.000000   \n",
       "1   8.0   8.0   7.0   5.0  10.0   4.0  11.0   9.0  10.0   7.0  ...   1.000000   \n",
       "2   6.0   5.0   8.0   1.0   5.0  10.0   7.0   4.0   6.0   7.0  ...   3.000000   \n",
       "3  10.0  17.0   9.0   5.0  13.0  16.0  16.0   8.0   9.0  15.0  ...   8.000000   \n",
       "4  25.0  23.0  18.0   8.0  36.0  30.0  26.0  16.0  43.0  43.0  ...  23.000000   \n",
       "5  27.0  41.0  23.0  21.0  31.0  34.0  37.0  34.0  45.0  38.0  ...  18.072161   \n",
       "\n",
       "   X1767  X1768  X1769  X1770  X1771  X1772  X1773  X1774      X1775  \n",
       "0   18.0   11.0   23.0    6.0   18.0   29.0    3.0   30.0  11.000000  \n",
       "1   18.0    9.0    3.0    4.0   12.0    5.0    7.0    9.0  13.000000  \n",
       "2   26.0   13.0   12.0    7.0   33.0    9.0    3.0   39.0  23.000000  \n",
       "3   43.0   59.0   37.0    9.0   48.0   41.0   12.0   62.0  52.000000  \n",
       "4   53.0   31.0   28.0   36.0   21.0   32.0    8.0   38.0  42.000000  \n",
       "5   26.0   36.0   40.0    9.0   14.0   59.0   17.0   22.0  43.372483  \n",
       "\n",
       "[6 rows x 1776 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r('''\n",
    "rpy2_observation_z[is.na(rpy2_observation_z)]<- NaN\n",
    "y <- data.matrix(rpy2_observation_z)\n",
    "Ya <- dineof(y)$Xa\n",
    "observation_z<-data.frame(Ya)\n",
    "head(observation_z)\n",
    "''')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "7Ld5tg99pbgU"
   },
   "source": [
    "## 補值過後的observation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "qKPj5H_ebpYd"
   },
   "source": [
    "#### R dataframe 轉換 pandas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-fh-KBY1bpYd",
    "outputId": "9eb3901e-1802-4c2f-9a50-a5bbe763a705"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X0</th>\n",
       "      <th>X1</th>\n",
       "      <th>X2</th>\n",
       "      <th>X3</th>\n",
       "      <th>X4</th>\n",
       "      <th>X5</th>\n",
       "      <th>X6</th>\n",
       "      <th>X7</th>\n",
       "      <th>X8</th>\n",
       "      <th>X9</th>\n",
       "      <th>...</th>\n",
       "      <th>X1766</th>\n",
       "      <th>X1767</th>\n",
       "      <th>X1768</th>\n",
       "      <th>X1769</th>\n",
       "      <th>X1770</th>\n",
       "      <th>X1771</th>\n",
       "      <th>X1772</th>\n",
       "      <th>X1773</th>\n",
       "      <th>X1774</th>\n",
       "      <th>X1775</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>28</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>13.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>30</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>...</td>\n",
       "      <td>17.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31</td>\n",
       "      <td>19.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>...</td>\n",
       "      <td>12.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32</td>\n",
       "      <td>27.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>...</td>\n",
       "      <td>25.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>27.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1776 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      X0    X1    X2    X3    X4    X5    X6    X7    X8    X9  ...  X1766  \\\n",
       "28  15.0  20.0  14.0  15.0  15.0  19.0  15.0  20.0  24.0  20.0  ...    4.0   \n",
       "29   8.0   9.0   8.0   5.0  11.0  10.0  10.0  14.0  15.0  12.0  ...    4.0   \n",
       "30   6.0   6.0   3.0   3.0   4.0   2.0   5.0   8.0   5.0   6.0  ...   17.0   \n",
       "31  19.0  19.0  24.0  15.0  18.0   7.0  12.0  12.0  21.0  18.0  ...   12.0   \n",
       "32  27.0  26.0  40.0  24.0  29.0  32.0  29.0  32.0  34.0  27.0  ...   25.0   \n",
       "\n",
       "    X1767  X1768  X1769  X1770  X1771  X1772  X1773  X1774  X1775  \n",
       "28   26.0    9.0   23.0    8.0   27.0   33.0   13.0   10.0   13.0  \n",
       "29   16.0    9.0   13.0    3.0   14.0   21.0   11.0    2.0    7.0  \n",
       "30   25.0   17.0   21.0   19.0   25.0   25.0    0.0   61.0   18.0  \n",
       "31   42.0   29.0   45.0   25.0   61.0   60.0    7.0   31.0   24.0  \n",
       "32   29.0   21.0   32.0   20.0   37.0   43.0    4.0   19.0   27.0  \n",
       "\n",
       "[5 rows x 1776 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.data('observation_z')\n",
    "df_nomissing = pd.DataFrame(r['observation_z'])\n",
    "df_nomissing.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DEzsMo_6pbga",
    "outputId": "eff416ee-8cf8-4af4-85f1-2de38e156321"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>5318</th>\n",
       "      <th>5319</th>\n",
       "      <th>5320</th>\n",
       "      <th>5321</th>\n",
       "      <th>5322</th>\n",
       "      <th>5323</th>\n",
       "      <th>5324</th>\n",
       "      <th>5325</th>\n",
       "      <th>5326</th>\n",
       "      <th>5327</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2019-09-29</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-09-30</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-02</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 5328 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            0     1     2     3     4     5     6     7     8     9     ...  \\\n",
       "2019-09-29   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "2019-09-30   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "2019-10-01   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "2019-10-02   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "2019-10-03   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "\n",
       "            5318  5319  5320  5321  5322  5323  5324  5325  5326  5327  \n",
       "2019-09-29   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "2019-09-30   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "2019-10-01   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "2019-10-02   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "2019-10-03   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "\n",
       "[5 rows x 5328 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index = pd.unique(df_observation['date'])\n",
    "observation_z = pd.DataFrame(columns=np.arange(station_num*time_step), data = np.nan, index = index)\n",
    "observation_z.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1Jdffgewpbgd",
    "outputId": "184ddb82-4bde-4830-84b0-5067a838444a"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>5318</th>\n",
       "      <th>5319</th>\n",
       "      <th>5320</th>\n",
       "      <th>5321</th>\n",
       "      <th>5322</th>\n",
       "      <th>5323</th>\n",
       "      <th>5324</th>\n",
       "      <th>5325</th>\n",
       "      <th>5326</th>\n",
       "      <th>5327</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2019-10-27</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>...</td>\n",
       "      <td>17.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-28</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>...</td>\n",
       "      <td>12.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-29</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>...</td>\n",
       "      <td>25.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>27.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-30</td>\n",
       "      <td>19.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-31</td>\n",
       "      <td>27.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 5328 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            0     1     2     3     4     5     6     7     8     9     ...  \\\n",
       "2019-10-27  15.0  20.0  14.0  15.0  15.0  19.0  15.0  20.0  24.0  20.0  ...   \n",
       "2019-10-28   8.0   9.0   8.0   5.0  11.0  10.0  10.0  14.0  15.0  12.0  ...   \n",
       "2019-10-29   6.0   6.0   3.0   3.0   4.0   2.0   5.0   8.0   5.0   6.0  ...   \n",
       "2019-10-30  19.0  19.0  24.0  15.0  18.0   7.0  12.0  12.0  21.0  18.0  ...   \n",
       "2019-10-31  27.0  26.0  40.0  24.0  29.0  32.0  29.0  32.0  34.0  27.0  ...   \n",
       "\n",
       "            5318  5319  5320  5321  5322  5323  5324  5325  5326  5327  \n",
       "2019-10-27  17.0  25.0  17.0  21.0  19.0  25.0  25.0   0.0  61.0  18.0  \n",
       "2019-10-28  12.0  42.0  29.0  45.0  25.0  61.0  60.0   7.0  31.0  24.0  \n",
       "2019-10-29  25.0  29.0  21.0  32.0  20.0  37.0  43.0   4.0  19.0  27.0  \n",
       "2019-10-30   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "2019-10-31   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "\n",
       "[5 rows x 5328 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "j = 1\n",
    "df_nomissing = np.array(df_nomissing)\n",
    "for i in range(len(observation_z)):\n",
    "    try:\n",
    "        observation_z.iloc[i,:]  = np.vstack(df_nomissing[i + np.arange(time_interval)]).flatten()\n",
    "    except:\n",
    "        observation_z.iloc[i,:]  = np.vstack([df_nomissing[i + np.arange(time_interval-j)],np.zeros_like(df_nomissing[:j]) * np.nan]).flatten()\n",
    "        j = j + 1\n",
    "observation_z.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Observation_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>5318</th>\n",
       "      <th>5319</th>\n",
       "      <th>5320</th>\n",
       "      <th>5321</th>\n",
       "      <th>5322</th>\n",
       "      <th>5323</th>\n",
       "      <th>5324</th>\n",
       "      <th>5325</th>\n",
       "      <th>5326</th>\n",
       "      <th>5327</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2019-10-27</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>...</td>\n",
       "      <td>17.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>18.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-28</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>...</td>\n",
       "      <td>12.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-29</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>...</td>\n",
       "      <td>25.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>27.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-30</td>\n",
       "      <td>19.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-31</td>\n",
       "      <td>27.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 5328 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            0     1     2     3     4     5     6     7     8     9     ...  \\\n",
       "2019-10-27  15.0  20.0  14.0  15.0  15.0  19.0  15.0  20.0  24.0  20.0  ...   \n",
       "2019-10-28   8.0   9.0   8.0   5.0  11.0  10.0  10.0  14.0  15.0  12.0  ...   \n",
       "2019-10-29   6.0   6.0   3.0   3.0   4.0   2.0   5.0   8.0   5.0   6.0  ...   \n",
       "2019-10-30  19.0  19.0  24.0  15.0  18.0   7.0  12.0  12.0  21.0  18.0  ...   \n",
       "2019-10-31  27.0  26.0  40.0  24.0  29.0  32.0  29.0  32.0  34.0  27.0  ...   \n",
       "\n",
       "            5318  5319  5320  5321  5322  5323  5324  5325  5326  5327  \n",
       "2019-10-27  17.0  25.0  17.0  21.0  19.0  25.0  25.0   0.0  61.0  18.0  \n",
       "2019-10-28  12.0  42.0  29.0  45.0  25.0  61.0  60.0   7.0  31.0  24.0  \n",
       "2019-10-29  25.0  29.0  21.0  32.0  20.0  37.0  43.0   4.0  19.0  27.0  \n",
       "2019-10-30   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "2019-10-31   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "\n",
       "[5 rows x 5328 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "observation_y = pd.DataFrame(columns=np.arange(station_num * time_step), data = np.nan,index = index)\n",
    "observation_y1 = np.array(observation_y1)\n",
    "j = 1\n",
    "df_nomissing = np.array(df_nomissing)\n",
    "for i in range(len(observation_z)):\n",
    "    try:\n",
    "        observation_y.iloc[i,:]  = np.vstack(observation_y1[i + np.arange(time_interval)]).flatten()\n",
    "    except:\n",
    "        observation_y.iloc[i,:]  = np.vstack([observation_y1[i + np.arange(time_interval-j)],np.zeros_like(observation_y1[:j]) * np.nan]).flatten()\n",
    "        j = j + 1\n",
    "observation_y.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "CdGlyMZjbpYi"
   },
   "source": [
    "# ----------------------------------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "p8ffuXZVbpYi"
   },
   "source": [
    "# Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "y3KM9kPQbpYk"
   },
   "outputs": [],
   "source": [
    "def getEveryDay(begin_date,end_date):\n",
    "    date_list = []\n",
    "    begin_date = datetime.strptime(begin_date, \"%Y-%m-%d\")\n",
    "    end_date = datetime.strptime(end_date,\"%Y-%m-%d\")\n",
    "    while begin_date <= end_date:\n",
    "        date_str = begin_date.strftime(\"%Y-%m-%d\")\n",
    "        date_list.append(date_str)\n",
    "        begin_date += timedelta(days=1)\n",
    "        \n",
    "    return date_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BrWAPKLKLQYm"
   },
   "outputs": [],
   "source": [
    "def data_cleaning_pred(df):\n",
    "    df['expdate'] = pd.to_datetime(df['expdate']).apply(lambda x:int(time.mktime(x.timetuple())))\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MZCEfeq7bpYm",
    "outputId": "28529a77-dd3c-4bb5-8e0f-20c7227875d0"
   },
   "outputs": [],
   "source": [
    "name = 'stat_v' \n",
    "for i in ver:\n",
    "    new_name = name + str(i) + '.txt'\n",
    "    prediction = pd.read_table(new_name)\n",
    "    if i == ver[0]:\n",
    "        new_prediction = prediction\n",
    "    else:\n",
    "        new_prediction = pd.concat([new_prediction, prediction], axis=0, sort = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Kro_nZcXLQYq"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\User\\Anaconda3\\envs\\tf2_rpy2\\lib\\site-packages\\ipykernel_launcher.py:15: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>hour</th>\n",
       "      <th>expdate</th>\n",
       "      <th>tau</th>\n",
       "      <th>stationID</th>\n",
       "      <th>calPMf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>595987</td>\n",
       "      <td>2019</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>23</td>\n",
       "      <td>2019-10-31</td>\n",
       "      <td>96</td>\n",
       "      <td>EPA084</td>\n",
       "      <td>0.159642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>595988</td>\n",
       "      <td>2019</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>23</td>\n",
       "      <td>2019-10-31</td>\n",
       "      <td>96</td>\n",
       "      <td>EPA101</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>595989</td>\n",
       "      <td>2019</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>23</td>\n",
       "      <td>2019-10-31</td>\n",
       "      <td>96</td>\n",
       "      <td>EPA102</td>\n",
       "      <td>0.250123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>595990</td>\n",
       "      <td>2019</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>23</td>\n",
       "      <td>2019-10-31</td>\n",
       "      <td>96</td>\n",
       "      <td>EPA104</td>\n",
       "      <td>0.521565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>595991</td>\n",
       "      <td>2019</td>\n",
       "      <td>11</td>\n",
       "      <td>4</td>\n",
       "      <td>23</td>\n",
       "      <td>2019-10-31</td>\n",
       "      <td>96</td>\n",
       "      <td>EPA110</td>\n",
       "      <td>18.928040</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        year  month  day  hour     expdate  tau stationID     calPMf\n",
       "595987  2019     11    4    23  2019-10-31   96    EPA084   0.159642\n",
       "595988  2019     11    4    23  2019-10-31   96    EPA101        NaN\n",
       "595989  2019     11    4    23  2019-10-31   96    EPA102   0.250123\n",
       "595990  2019     11    4    23  2019-10-31   96    EPA104   0.521565\n",
       "595991  2019     11    4    23  2019-10-31   96    EPA110  18.928040"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_prediction = data_cleaning_pred(new_prediction)\n",
    "file_name = 'corr_old_preciction_' + str(time_step) + '.txt'\n",
    "\n",
    "try:\n",
    "    old_prediction = pd.read_table(file_name)\n",
    "    old_prediction = data_cleaning_pred(old_prediction)\n",
    "    old_end = old_prediction['expdate'].iloc[-1] + 3600\n",
    "    new_prediction = new_prediction[new_prediction['expdate']>=old_end]\n",
    "    df_prediction = pd.concat([old_prediction, new_prediction], axis = 0, ignore_index=True)\n",
    "    \n",
    "except IOError:\n",
    "    df_prediction = new_prediction\n",
    "    \n",
    "df_prediction = df_prediction[df_prediction['expdate'].between(start - 3600*24, end_pre)] #end_pre - 3600 * 24\n",
    "df_prediction['expdate'] = df_prediction['expdate'].apply(lambda x:time.strftime(\"%Y-%m-%d\", time.localtime(x))) \n",
    "df_prediction.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "mi_bpTQ0bpYp"
   },
   "source": [
    "## 拿取預測值、測站移除"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hcgYvtnPbpYq",
    "outputId": "13500ae0-9e9f-495c-a8a9-d9e66eebb639"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>hour</th>\n",
       "      <th>expdate</th>\n",
       "      <th>tau</th>\n",
       "      <th>stationID</th>\n",
       "      <th>calPMf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>297240</td>\n",
       "      <td>2019</td>\n",
       "      <td>9</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-09-29</td>\n",
       "      <td>1</td>\n",
       "      <td>EPA001</td>\n",
       "      <td>10.384020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>297241</td>\n",
       "      <td>2019</td>\n",
       "      <td>9</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-09-29</td>\n",
       "      <td>1</td>\n",
       "      <td>EPA002</td>\n",
       "      <td>12.222376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>297242</td>\n",
       "      <td>2019</td>\n",
       "      <td>9</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-09-29</td>\n",
       "      <td>1</td>\n",
       "      <td>EPA003</td>\n",
       "      <td>5.971256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>297243</td>\n",
       "      <td>2019</td>\n",
       "      <td>9</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-09-29</td>\n",
       "      <td>1</td>\n",
       "      <td>EPA004</td>\n",
       "      <td>11.441598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>297244</td>\n",
       "      <td>2019</td>\n",
       "      <td>9</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-09-29</td>\n",
       "      <td>1</td>\n",
       "      <td>EPA005</td>\n",
       "      <td>10.245286</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        year  month  day  hour     expdate  tau stationID     calPMf\n",
       "297240  2019      9   30     0  2019-09-29    1    EPA001  10.384020\n",
       "297241  2019      9   30     0  2019-09-29    1    EPA002  12.222376\n",
       "297242  2019      9   30     0  2019-09-29    1    EPA003   5.971256\n",
       "297243  2019      9   30     0  2019-09-29    1    EPA004  11.441598\n",
       "297244  2019      9   30     0  2019-09-29    1    EPA005  10.245286"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_prediction = df_prediction[df_prediction['tau']!=0]\n",
    "df_prediction = df_prediction[df_prediction['tau'] <= time_step]\n",
    "\n",
    "df_prediction = df_prediction[df_prediction['stationID']!='EPA101']\n",
    "df_prediction = df_prediction[df_prediction['stationID']!='EPA084']\n",
    "df_prediction = df_prediction[df_prediction['stationID']!='EPA104']\n",
    "df_prediction.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>hour</th>\n",
       "      <th>expdate</th>\n",
       "      <th>tau</th>\n",
       "      <th>stationID</th>\n",
       "      <th>calPMf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>594136</td>\n",
       "      <td>2019</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>23</td>\n",
       "      <td>2019-10-31</td>\n",
       "      <td>72</td>\n",
       "      <td>EPA071</td>\n",
       "      <td>51.187390</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>594137</td>\n",
       "      <td>2019</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>23</td>\n",
       "      <td>2019-10-31</td>\n",
       "      <td>72</td>\n",
       "      <td>EPA072</td>\n",
       "      <td>14.749970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>594138</td>\n",
       "      <td>2019</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>23</td>\n",
       "      <td>2019-10-31</td>\n",
       "      <td>72</td>\n",
       "      <td>EPA080</td>\n",
       "      <td>0.360062</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>594141</td>\n",
       "      <td>2019</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>23</td>\n",
       "      <td>2019-10-31</td>\n",
       "      <td>72</td>\n",
       "      <td>EPA102</td>\n",
       "      <td>0.444094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>594143</td>\n",
       "      <td>2019</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>23</td>\n",
       "      <td>2019-10-31</td>\n",
       "      <td>72</td>\n",
       "      <td>EPA110</td>\n",
       "      <td>17.873880</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        year  month  day  hour     expdate  tau stationID     calPMf\n",
       "594136  2019     11    3    23  2019-10-31   72    EPA071  51.187390\n",
       "594137  2019     11    3    23  2019-10-31   72    EPA072  14.749970\n",
       "594138  2019     11    3    23  2019-10-31   72    EPA080   0.360062\n",
       "594141  2019     11    3    23  2019-10-31   72    EPA102   0.444094\n",
       "594143  2019     11    3    23  2019-10-31   72    EPA110  17.873880"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_prediction.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "bCWLc0B6bpYr"
   },
   "source": [
    "## dateTime為預測的時間"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "04B4UsrxbpYs",
    "outputId": "18457e95-643b-4800-89ec-c1f744299f7e"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>hour</th>\n",
       "      <th>expdate</th>\n",
       "      <th>tau</th>\n",
       "      <th>stationID</th>\n",
       "      <th>calPMf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>297240</td>\n",
       "      <td>2019</td>\n",
       "      <td>9</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-09-29</td>\n",
       "      <td>1</td>\n",
       "      <td>EPA001</td>\n",
       "      <td>10.384020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>297241</td>\n",
       "      <td>2019</td>\n",
       "      <td>9</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-09-29</td>\n",
       "      <td>1</td>\n",
       "      <td>EPA002</td>\n",
       "      <td>12.222376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>297242</td>\n",
       "      <td>2019</td>\n",
       "      <td>9</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-09-29</td>\n",
       "      <td>1</td>\n",
       "      <td>EPA003</td>\n",
       "      <td>5.971256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>297243</td>\n",
       "      <td>2019</td>\n",
       "      <td>9</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-09-29</td>\n",
       "      <td>1</td>\n",
       "      <td>EPA004</td>\n",
       "      <td>11.441598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>297244</td>\n",
       "      <td>2019</td>\n",
       "      <td>9</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>2019-09-29</td>\n",
       "      <td>1</td>\n",
       "      <td>EPA005</td>\n",
       "      <td>10.245286</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        year  month  day  hour     expdate  tau stationID     calPMf\n",
       "297240  2019      9   30     0  2019-09-29    1    EPA001  10.384020\n",
       "297241  2019      9   30     0  2019-09-29    1    EPA002  12.222376\n",
       "297242  2019      9   30     0  2019-09-29    1    EPA003   5.971256\n",
       "297243  2019      9   30     0  2019-09-29    1    EPA004  11.441598\n",
       "297244  2019      9   30     0  2019-09-29    1    EPA005  10.245286"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_prediction['expdate'] = pd.to_datetime(df_prediction['expdate']).apply(lambda x:int(time.mktime(x.timetuple())))\n",
    "df_prediction['expdate'] = df_prediction['expdate'].apply(lambda x:time.strftime(\"%Y-%m-%d\",time.localtime(x))) \n",
    "df_prediction.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "HujL2gkxbpYt"
   },
   "source": [
    "## 日期、時間缺漏值進行填補"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "v_cZ0kIMbpYu"
   },
   "outputs": [],
   "source": [
    "def otherStyleTime(timestamp):\n",
    "    timeArray = time.localtime(timestamp)\n",
    "    date = time.strftime(\"%Y-%m-%d\", timeArray)\n",
    "    return(date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "cbIGMLbnbpYv"
   },
   "outputs": [],
   "source": [
    "start_pre = otherStyleTime(start)\n",
    "end_pre = otherStyleTime(end_pre )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "x1GWaqtdbpYx",
    "outputId": "c260ff86-ff54-4cbc-c268-4b8d27008edf"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------prediction 補缺失日期開始------\n",
      "stations: EPA001\n",
      "stations: EPA002\n",
      "stations: EPA003\n",
      "stations: EPA004\n",
      "stations: EPA005\n",
      "stations: EPA006\n",
      "stations: EPA007\n",
      "stations: EPA008\n",
      "stations: EPA009\n",
      "stations: EPA010\n",
      "stations: EPA011\n",
      "stations: EPA012\n",
      "stations: EPA013\n",
      "stations: EPA014\n",
      "stations: EPA015\n",
      "stations: EPA016\n",
      "stations: EPA017\n",
      "stations: EPA018\n",
      "stations: EPA019\n",
      "stations: EPA020\n",
      "stations: EPA021\n",
      "stations: EPA022\n",
      "stations: EPA023\n",
      "stations: EPA024\n",
      "stations: EPA025\n",
      "stations: EPA026\n",
      "stations: EPA027\n",
      "stations: EPA028\n",
      "stations: EPA029\n",
      "stations: EPA030\n",
      "stations: EPA031\n",
      "stations: EPA032\n",
      "stations: EPA033\n",
      "stations: EPA034\n",
      "stations: EPA035\n",
      "stations: EPA036\n",
      "stations: EPA037\n",
      "stations: EPA038\n",
      "stations: EPA039\n",
      "stations: EPA040\n",
      "stations: EPA041\n",
      "stations: EPA042\n",
      "stations: EPA043\n",
      "stations: EPA044\n",
      "stations: EPA045\n",
      "stations: EPA046\n",
      "stations: EPA047\n",
      "stations: EPA048\n",
      "stations: EPA049\n",
      "stations: EPA050\n",
      "stations: EPA051\n",
      "stations: EPA052\n",
      "stations: EPA053\n",
      "stations: EPA054\n",
      "stations: EPA056\n",
      "stations: EPA057\n",
      "stations: EPA058\n",
      "stations: EPA059\n",
      "stations: EPA060\n",
      "stations: EPA061\n",
      "stations: EPA062\n",
      "stations: EPA063\n",
      "stations: EPA064\n",
      "stations: EPA065\n",
      "stations: EPA066\n",
      "stations: EPA067\n",
      "stations: EPA068\n",
      "stations: EPA069\n",
      "stations: EPA070\n",
      "stations: EPA071\n",
      "stations: EPA072\n",
      "stations: EPA080\n",
      "stations: EPA102\n",
      "stations: EPA110\n"
     ]
    }
   ],
   "source": [
    "stations = df_prediction['stationID'].unique()\n",
    "all_expdate = getEveryDay(start_pre, end_pre)\n",
    "print('------prediction 補缺失日期開始------')\n",
    "for name in stations:\n",
    "    \n",
    "    # 補一整天皆缺失的值\n",
    "    unnormal = list(df_prediction[df_prediction['stationID'] == name]['expdate'])\n",
    "    dates = list(set(all_expdate).difference(set(unnormal)))\n",
    "\n",
    "    for date in dates:\n",
    "        df_add = df_prediction[: time_step].copy()\n",
    "        df_add['expdate'] = date\n",
    "        df_add['hour'] = list(np.arange(24))*time_interval\n",
    "        df_add['stationID'] = name\n",
    "        df_add['tau'] = np.arange(time_step) + 1\n",
    "        \n",
    "        day = []\n",
    "        month = []\n",
    "        year = []\n",
    "        for t in range(1, time_interval+1):\n",
    "            day += [(pd.to_datetime(date) + timedelta(days=t)).day] * 24 \n",
    "            month += [(pd.to_datetime(date) + timedelta(days=t)).month] * 24 \n",
    "            year += [(pd.to_datetime(date) + timedelta(days=t)).year] * 24 \n",
    "        df_add['day'] = day\n",
    "        df_add['month'] = month\n",
    "        df_add['year'] = year\n",
    "        df_add['calPMf'] = np.nan\n",
    "        df_prediction = pd.concat([df_prediction, df_add])\n",
    "\n",
    "    df_prediction.sort_values(by=['expdate', 'tau', 'stationID'], inplace = True)\n",
    "\n",
    "    # 補部分缺失的值\n",
    "    if sum(pd.value_counts(df_prediction[df_prediction['stationID']==name]['expdate'])!=time_step)>0:\n",
    "        #print(name, date)\n",
    "        count = pd.value_counts(df_prediction[df_prediction['stationID']==name]['expdate']).values\n",
    "        index = pd.value_counts(df_prediction[df_prediction['stationID']==name]['expdate']).index\n",
    "        for expdate in index[count!=time_step]:\n",
    "            all_hour = np.arange(time_step) + 1\n",
    "            unnormal = df_prediction[df_prediction['stationID']==name][df_prediction['expdate']==expdate]['tau']\n",
    "            taus = list(set(all_hour).difference(set(unnormal)))\n",
    "            num = len(hours)\n",
    "\n",
    "            df_add = df_prediction[: len(taus)].copy()\n",
    "            df_add['expdate'] = expdate  \n",
    "            df_add['hour'] = taus % 24 \n",
    "            df_add['stationID'] = name\n",
    "            df_add['tau'] = taus\n",
    "            df_add['day'] = (pd.to_datetime(date) + timedelta(hours=23+taus)).day\n",
    "            df_add['month'] = (pd.to_datetime(date) + timedelta(hours=23+taus)).month\n",
    "            df_add['year'] = (pd.to_datetime(date) + timedelta(hours=23+taus)).year\n",
    "            df_add['calPMf'] = np.nan\n",
    "            df_prediction = pd.concat([df_prediction, df_add])\n",
    "\n",
    "    df_prediction.sort_values(by=['expdate', 'tau', 'stationID'], inplace = True)\n",
    "    print('stations:', name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EOKIZP19ni7q"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "prediction補缺失日期成功\n"
     ]
    }
   ],
   "source": [
    "if len(np.unique(pd.value_counts(df_prediction['stationID'])))==1:\n",
    "    print(\"prediction補缺失日期成功\")\n",
    "else:\n",
    "    print(\"prediction補缺失日期失敗\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "apFH93M3bpYz",
    "outputId": "d00c7418-b351-46e8-d522-d9e6cb19836c"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>expdate</th>\n",
       "      <th>stationID</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>hour</th>\n",
       "      <th>tau</th>\n",
       "      <th>calPMf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>297240</td>\n",
       "      <td>2019-09-29</td>\n",
       "      <td>EPA001</td>\n",
       "      <td>2019</td>\n",
       "      <td>9</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>10.384020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>297241</td>\n",
       "      <td>2019-09-29</td>\n",
       "      <td>EPA002</td>\n",
       "      <td>2019</td>\n",
       "      <td>9</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>12.222376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>297242</td>\n",
       "      <td>2019-09-29</td>\n",
       "      <td>EPA003</td>\n",
       "      <td>2019</td>\n",
       "      <td>9</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>5.971256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>297243</td>\n",
       "      <td>2019-09-29</td>\n",
       "      <td>EPA004</td>\n",
       "      <td>2019</td>\n",
       "      <td>9</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>11.441598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>297244</td>\n",
       "      <td>2019-09-29</td>\n",
       "      <td>EPA005</td>\n",
       "      <td>2019</td>\n",
       "      <td>9</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>10.245286</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           expdate stationID  year  month  day  hour  tau     calPMf\n",
       "297240  2019-09-29    EPA001  2019      9   30     0    1  10.384020\n",
       "297241  2019-09-29    EPA002  2019      9   30     0    1  12.222376\n",
       "297242  2019-09-29    EPA003  2019      9   30     0    1   5.971256\n",
       "297243  2019-09-29    EPA004  2019      9   30     0    1  11.441598\n",
       "297244  2019-09-29    EPA005  2019      9   30     0    1  10.245286"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_prediction = df_prediction[['expdate','stationID','year','month','day','hour','tau','calPMf']]\n",
    "df_prediction.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "a8R4pbx1cVvT"
   },
   "outputs": [],
   "source": [
    "df_prediction.to_csv(file_name, sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "T3PaGA6cbpY3"
   },
   "source": [
    "## 將tau = 1的日期設為index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iQvKWr-abpY5",
    "outputId": "e810ea5c-4d48-4f6b-9434-b38c400272be"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>5318</th>\n",
       "      <th>5319</th>\n",
       "      <th>5320</th>\n",
       "      <th>5321</th>\n",
       "      <th>5322</th>\n",
       "      <th>5323</th>\n",
       "      <th>5324</th>\n",
       "      <th>5325</th>\n",
       "      <th>5326</th>\n",
       "      <th>5327</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2019-09-30</td>\n",
       "      <td>10.384020</td>\n",
       "      <td>12.222376</td>\n",
       "      <td>5.971256</td>\n",
       "      <td>11.441598</td>\n",
       "      <td>10.245286</td>\n",
       "      <td>14.770390</td>\n",
       "      <td>14.752220</td>\n",
       "      <td>16.288942</td>\n",
       "      <td>15.054422</td>\n",
       "      <td>9.530121</td>\n",
       "      <td>...</td>\n",
       "      <td>6.551325</td>\n",
       "      <td>6.648709</td>\n",
       "      <td>6.888054</td>\n",
       "      <td>15.446540</td>\n",
       "      <td>6.128881</td>\n",
       "      <td>71.70889</td>\n",
       "      <td>2.563082</td>\n",
       "      <td>0.654434</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.045380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-01</td>\n",
       "      <td>2.303247</td>\n",
       "      <td>1.167996</td>\n",
       "      <td>3.119260</td>\n",
       "      <td>1.364694</td>\n",
       "      <td>2.355424</td>\n",
       "      <td>3.042923</td>\n",
       "      <td>3.324206</td>\n",
       "      <td>9.170274</td>\n",
       "      <td>2.758917</td>\n",
       "      <td>1.507425</td>\n",
       "      <td>...</td>\n",
       "      <td>8.073354</td>\n",
       "      <td>9.741949</td>\n",
       "      <td>19.012670</td>\n",
       "      <td>36.024790</td>\n",
       "      <td>13.535630</td>\n",
       "      <td>46.60602</td>\n",
       "      <td>3.803456</td>\n",
       "      <td>0.691928</td>\n",
       "      <td>1.578971</td>\n",
       "      <td>1.534836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-02</td>\n",
       "      <td>18.529622</td>\n",
       "      <td>16.354433</td>\n",
       "      <td>18.980730</td>\n",
       "      <td>14.328082</td>\n",
       "      <td>18.858808</td>\n",
       "      <td>20.995500</td>\n",
       "      <td>22.920832</td>\n",
       "      <td>18.492727</td>\n",
       "      <td>28.264308</td>\n",
       "      <td>17.527233</td>\n",
       "      <td>...</td>\n",
       "      <td>8.417409</td>\n",
       "      <td>23.461110</td>\n",
       "      <td>33.428940</td>\n",
       "      <td>11.227320</td>\n",
       "      <td>32.616840</td>\n",
       "      <td>36.36791</td>\n",
       "      <td>3.497346</td>\n",
       "      <td>0.254489</td>\n",
       "      <td>34.839195</td>\n",
       "      <td>29.089000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-03</td>\n",
       "      <td>25.217936</td>\n",
       "      <td>27.691776</td>\n",
       "      <td>26.745528</td>\n",
       "      <td>24.618408</td>\n",
       "      <td>33.900899</td>\n",
       "      <td>35.589010</td>\n",
       "      <td>33.690254</td>\n",
       "      <td>38.341235</td>\n",
       "      <td>34.809658</td>\n",
       "      <td>29.657188</td>\n",
       "      <td>...</td>\n",
       "      <td>7.866346</td>\n",
       "      <td>47.601430</td>\n",
       "      <td>16.523700</td>\n",
       "      <td>1.502065</td>\n",
       "      <td>15.735460</td>\n",
       "      <td>147.13415</td>\n",
       "      <td>1.472838</td>\n",
       "      <td>0.353094</td>\n",
       "      <td>164.752345</td>\n",
       "      <td>13.292680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-04</td>\n",
       "      <td>38.854236</td>\n",
       "      <td>36.755066</td>\n",
       "      <td>35.572574</td>\n",
       "      <td>34.110252</td>\n",
       "      <td>48.683100</td>\n",
       "      <td>52.354600</td>\n",
       "      <td>49.098145</td>\n",
       "      <td>44.835977</td>\n",
       "      <td>43.202534</td>\n",
       "      <td>41.142932</td>\n",
       "      <td>...</td>\n",
       "      <td>2.602429</td>\n",
       "      <td>5.945044</td>\n",
       "      <td>12.129320</td>\n",
       "      <td>1.327380</td>\n",
       "      <td>7.761159</td>\n",
       "      <td>21.69153</td>\n",
       "      <td>2.312886</td>\n",
       "      <td>0.734246</td>\n",
       "      <td>54.006198</td>\n",
       "      <td>20.799630</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 5328 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0          1          2          3          4          5     \\\n",
       "2019-09-30  10.384020  12.222376   5.971256  11.441598  10.245286  14.770390   \n",
       "2019-10-01   2.303247   1.167996   3.119260   1.364694   2.355424   3.042923   \n",
       "2019-10-02  18.529622  16.354433  18.980730  14.328082  18.858808  20.995500   \n",
       "2019-10-03  25.217936  27.691776  26.745528  24.618408  33.900899  35.589010   \n",
       "2019-10-04  38.854236  36.755066  35.572574  34.110252  48.683100  52.354600   \n",
       "\n",
       "                 6          7          8          9     ...      5318  \\\n",
       "2019-09-30  14.752220  16.288942  15.054422   9.530121  ...  6.551325   \n",
       "2019-10-01   3.324206   9.170274   2.758917   1.507425  ...  8.073354   \n",
       "2019-10-02  22.920832  18.492727  28.264308  17.527233  ...  8.417409   \n",
       "2019-10-03  33.690254  38.341235  34.809658  29.657188  ...  7.866346   \n",
       "2019-10-04  49.098145  44.835977  43.202534  41.142932  ...  2.602429   \n",
       "\n",
       "                 5319       5320       5321       5322       5323      5324  \\\n",
       "2019-09-30   6.648709   6.888054  15.446540   6.128881   71.70889  2.563082   \n",
       "2019-10-01   9.741949  19.012670  36.024790  13.535630   46.60602  3.803456   \n",
       "2019-10-02  23.461110  33.428940  11.227320  32.616840   36.36791  3.497346   \n",
       "2019-10-03  47.601430  16.523700   1.502065  15.735460  147.13415  1.472838   \n",
       "2019-10-04   5.945044  12.129320   1.327380   7.761159   21.69153  2.312886   \n",
       "\n",
       "                5325        5326       5327  \n",
       "2019-09-30  0.654434         NaN   8.045380  \n",
       "2019-10-01  0.691928    1.578971   1.534836  \n",
       "2019-10-02  0.254489   34.839195  29.089000  \n",
       "2019-10-03  0.353094  164.752345  13.292680  \n",
       "2019-10-04  0.734246   54.006198  20.799630  \n",
       "\n",
       "[5 rows x 5328 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pred = pd.DataFrame(np.array(df_prediction['calPMf']).reshape([-1, time_step * station_num]))\n",
    "df_pred.index = pd.to_datetime(df_prediction['expdate'].unique(), errors='coerce') + timedelta(days=1)\n",
    "index = df_pred.index \n",
    "df_pred.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>5318</th>\n",
       "      <th>5319</th>\n",
       "      <th>5320</th>\n",
       "      <th>5321</th>\n",
       "      <th>5322</th>\n",
       "      <th>5323</th>\n",
       "      <th>5324</th>\n",
       "      <th>5325</th>\n",
       "      <th>5326</th>\n",
       "      <th>5327</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2019-10-28</td>\n",
       "      <td>13.427509</td>\n",
       "      <td>13.446619</td>\n",
       "      <td>12.398126</td>\n",
       "      <td>14.050098</td>\n",
       "      <td>16.355950</td>\n",
       "      <td>20.087620</td>\n",
       "      <td>16.750780</td>\n",
       "      <td>18.338449</td>\n",
       "      <td>19.555760</td>\n",
       "      <td>16.339273</td>\n",
       "      <td>...</td>\n",
       "      <td>8.265961</td>\n",
       "      <td>7.216942</td>\n",
       "      <td>7.790610</td>\n",
       "      <td>14.271270</td>\n",
       "      <td>4.566690</td>\n",
       "      <td>119.616840</td>\n",
       "      <td>4.381921</td>\n",
       "      <td>0.757728</td>\n",
       "      <td>11.389323</td>\n",
       "      <td>5.244024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-29</td>\n",
       "      <td>5.632875</td>\n",
       "      <td>4.230715</td>\n",
       "      <td>0.269621</td>\n",
       "      <td>2.059012</td>\n",
       "      <td>11.498530</td>\n",
       "      <td>11.264280</td>\n",
       "      <td>6.891853</td>\n",
       "      <td>10.196930</td>\n",
       "      <td>4.680984</td>\n",
       "      <td>2.894081</td>\n",
       "      <td>...</td>\n",
       "      <td>3.410336</td>\n",
       "      <td>7.111055</td>\n",
       "      <td>10.344857</td>\n",
       "      <td>65.649088</td>\n",
       "      <td>9.054913</td>\n",
       "      <td>26.173115</td>\n",
       "      <td>16.586438</td>\n",
       "      <td>1.491971</td>\n",
       "      <td>2.582207</td>\n",
       "      <td>9.006240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-30</td>\n",
       "      <td>1.289931</td>\n",
       "      <td>19.258952</td>\n",
       "      <td>0.432383</td>\n",
       "      <td>14.901676</td>\n",
       "      <td>14.629965</td>\n",
       "      <td>15.920009</td>\n",
       "      <td>17.553282</td>\n",
       "      <td>19.671755</td>\n",
       "      <td>11.698523</td>\n",
       "      <td>16.916361</td>\n",
       "      <td>...</td>\n",
       "      <td>2.414034</td>\n",
       "      <td>6.646315</td>\n",
       "      <td>4.300676</td>\n",
       "      <td>23.151850</td>\n",
       "      <td>3.250110</td>\n",
       "      <td>26.905080</td>\n",
       "      <td>18.146720</td>\n",
       "      <td>0.174710</td>\n",
       "      <td>0.259247</td>\n",
       "      <td>4.808150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-31</td>\n",
       "      <td>1.264438</td>\n",
       "      <td>29.788514</td>\n",
       "      <td>0.255462</td>\n",
       "      <td>22.720576</td>\n",
       "      <td>12.146773</td>\n",
       "      <td>22.353600</td>\n",
       "      <td>29.110149</td>\n",
       "      <td>34.981333</td>\n",
       "      <td>34.254231</td>\n",
       "      <td>25.641742</td>\n",
       "      <td>...</td>\n",
       "      <td>3.009589</td>\n",
       "      <td>3.709717</td>\n",
       "      <td>7.282560</td>\n",
       "      <td>23.191280</td>\n",
       "      <td>3.551979</td>\n",
       "      <td>36.680530</td>\n",
       "      <td>26.542503</td>\n",
       "      <td>0.535734</td>\n",
       "      <td>0.256727</td>\n",
       "      <td>5.043830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-11-01</td>\n",
       "      <td>2.259185</td>\n",
       "      <td>19.170722</td>\n",
       "      <td>0.316616</td>\n",
       "      <td>20.142869</td>\n",
       "      <td>9.882625</td>\n",
       "      <td>20.470040</td>\n",
       "      <td>20.447631</td>\n",
       "      <td>22.565321</td>\n",
       "      <td>22.329302</td>\n",
       "      <td>19.088418</td>\n",
       "      <td>...</td>\n",
       "      <td>3.513030</td>\n",
       "      <td>6.379925</td>\n",
       "      <td>11.479990</td>\n",
       "      <td>27.022449</td>\n",
       "      <td>3.002634</td>\n",
       "      <td>51.187390</td>\n",
       "      <td>14.749970</td>\n",
       "      <td>0.360062</td>\n",
       "      <td>0.444094</td>\n",
       "      <td>17.873880</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 5328 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0          1          2          3          4          5     \\\n",
       "2019-10-28  13.427509  13.446619  12.398126  14.050098  16.355950  20.087620   \n",
       "2019-10-29   5.632875   4.230715   0.269621   2.059012  11.498530  11.264280   \n",
       "2019-10-30   1.289931  19.258952   0.432383  14.901676  14.629965  15.920009   \n",
       "2019-10-31   1.264438  29.788514   0.255462  22.720576  12.146773  22.353600   \n",
       "2019-11-01   2.259185  19.170722   0.316616  20.142869   9.882625  20.470040   \n",
       "\n",
       "                 6          7          8          9     ...      5318  \\\n",
       "2019-10-28  16.750780  18.338449  19.555760  16.339273  ...  8.265961   \n",
       "2019-10-29   6.891853  10.196930   4.680984   2.894081  ...  3.410336   \n",
       "2019-10-30  17.553282  19.671755  11.698523  16.916361  ...  2.414034   \n",
       "2019-10-31  29.110149  34.981333  34.254231  25.641742  ...  3.009589   \n",
       "2019-11-01  20.447631  22.565321  22.329302  19.088418  ...  3.513030   \n",
       "\n",
       "                5319       5320       5321      5322        5323       5324  \\\n",
       "2019-10-28  7.216942   7.790610  14.271270  4.566690  119.616840   4.381921   \n",
       "2019-10-29  7.111055  10.344857  65.649088  9.054913   26.173115  16.586438   \n",
       "2019-10-30  6.646315   4.300676  23.151850  3.250110   26.905080  18.146720   \n",
       "2019-10-31  3.709717   7.282560  23.191280  3.551979   36.680530  26.542503   \n",
       "2019-11-01  6.379925  11.479990  27.022449  3.002634   51.187390  14.749970   \n",
       "\n",
       "                5325       5326       5327  \n",
       "2019-10-28  0.757728  11.389323   5.244024  \n",
       "2019-10-29  1.491971   2.582207   9.006240  \n",
       "2019-10-30  0.174710   0.259247   4.808150  \n",
       "2019-10-31  0.535734   0.256727   5.043830  \n",
       "2019-11-01  0.360062   0.444094  17.873880  \n",
       "\n",
       "[5 rows x 5328 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pred.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "KvIT6TW1bpY7"
   },
   "source": [
    "## 將一整天為NAN的值刪掉"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BeYV4XDJbpY7",
    "outputId": "be92a513-0311-44c3-f7c4-72c2b84cebe1"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>5318</th>\n",
       "      <th>5319</th>\n",
       "      <th>5320</th>\n",
       "      <th>5321</th>\n",
       "      <th>5322</th>\n",
       "      <th>5323</th>\n",
       "      <th>5324</th>\n",
       "      <th>5325</th>\n",
       "      <th>5326</th>\n",
       "      <th>5327</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2019-09-30</td>\n",
       "      <td>10.384020</td>\n",
       "      <td>12.222376</td>\n",
       "      <td>5.971256</td>\n",
       "      <td>11.441598</td>\n",
       "      <td>10.245286</td>\n",
       "      <td>14.770390</td>\n",
       "      <td>14.752220</td>\n",
       "      <td>16.288942</td>\n",
       "      <td>15.054422</td>\n",
       "      <td>9.530121</td>\n",
       "      <td>...</td>\n",
       "      <td>6.551325</td>\n",
       "      <td>6.648709</td>\n",
       "      <td>6.888054</td>\n",
       "      <td>15.446540</td>\n",
       "      <td>6.128881</td>\n",
       "      <td>71.70889</td>\n",
       "      <td>2.563082</td>\n",
       "      <td>0.654434</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.045380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-01</td>\n",
       "      <td>2.303247</td>\n",
       "      <td>1.167996</td>\n",
       "      <td>3.119260</td>\n",
       "      <td>1.364694</td>\n",
       "      <td>2.355424</td>\n",
       "      <td>3.042923</td>\n",
       "      <td>3.324206</td>\n",
       "      <td>9.170274</td>\n",
       "      <td>2.758917</td>\n",
       "      <td>1.507425</td>\n",
       "      <td>...</td>\n",
       "      <td>8.073354</td>\n",
       "      <td>9.741949</td>\n",
       "      <td>19.012670</td>\n",
       "      <td>36.024790</td>\n",
       "      <td>13.535630</td>\n",
       "      <td>46.60602</td>\n",
       "      <td>3.803456</td>\n",
       "      <td>0.691928</td>\n",
       "      <td>1.578971</td>\n",
       "      <td>1.534836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-02</td>\n",
       "      <td>18.529622</td>\n",
       "      <td>16.354433</td>\n",
       "      <td>18.980730</td>\n",
       "      <td>14.328082</td>\n",
       "      <td>18.858808</td>\n",
       "      <td>20.995500</td>\n",
       "      <td>22.920832</td>\n",
       "      <td>18.492727</td>\n",
       "      <td>28.264308</td>\n",
       "      <td>17.527233</td>\n",
       "      <td>...</td>\n",
       "      <td>8.417409</td>\n",
       "      <td>23.461110</td>\n",
       "      <td>33.428940</td>\n",
       "      <td>11.227320</td>\n",
       "      <td>32.616840</td>\n",
       "      <td>36.36791</td>\n",
       "      <td>3.497346</td>\n",
       "      <td>0.254489</td>\n",
       "      <td>34.839195</td>\n",
       "      <td>29.089000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-03</td>\n",
       "      <td>25.217936</td>\n",
       "      <td>27.691776</td>\n",
       "      <td>26.745528</td>\n",
       "      <td>24.618408</td>\n",
       "      <td>33.900899</td>\n",
       "      <td>35.589010</td>\n",
       "      <td>33.690254</td>\n",
       "      <td>38.341235</td>\n",
       "      <td>34.809658</td>\n",
       "      <td>29.657188</td>\n",
       "      <td>...</td>\n",
       "      <td>7.866346</td>\n",
       "      <td>47.601430</td>\n",
       "      <td>16.523700</td>\n",
       "      <td>1.502065</td>\n",
       "      <td>15.735460</td>\n",
       "      <td>147.13415</td>\n",
       "      <td>1.472838</td>\n",
       "      <td>0.353094</td>\n",
       "      <td>164.752345</td>\n",
       "      <td>13.292680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-04</td>\n",
       "      <td>38.854236</td>\n",
       "      <td>36.755066</td>\n",
       "      <td>35.572574</td>\n",
       "      <td>34.110252</td>\n",
       "      <td>48.683100</td>\n",
       "      <td>52.354600</td>\n",
       "      <td>49.098145</td>\n",
       "      <td>44.835977</td>\n",
       "      <td>43.202534</td>\n",
       "      <td>41.142932</td>\n",
       "      <td>...</td>\n",
       "      <td>2.602429</td>\n",
       "      <td>5.945044</td>\n",
       "      <td>12.129320</td>\n",
       "      <td>1.327380</td>\n",
       "      <td>7.761159</td>\n",
       "      <td>21.69153</td>\n",
       "      <td>2.312886</td>\n",
       "      <td>0.734246</td>\n",
       "      <td>54.006198</td>\n",
       "      <td>20.799630</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 5328 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0          1          2          3          4          5     \\\n",
       "2019-09-30  10.384020  12.222376   5.971256  11.441598  10.245286  14.770390   \n",
       "2019-10-01   2.303247   1.167996   3.119260   1.364694   2.355424   3.042923   \n",
       "2019-10-02  18.529622  16.354433  18.980730  14.328082  18.858808  20.995500   \n",
       "2019-10-03  25.217936  27.691776  26.745528  24.618408  33.900899  35.589010   \n",
       "2019-10-04  38.854236  36.755066  35.572574  34.110252  48.683100  52.354600   \n",
       "\n",
       "                 6          7          8          9     ...      5318  \\\n",
       "2019-09-30  14.752220  16.288942  15.054422   9.530121  ...  6.551325   \n",
       "2019-10-01   3.324206   9.170274   2.758917   1.507425  ...  8.073354   \n",
       "2019-10-02  22.920832  18.492727  28.264308  17.527233  ...  8.417409   \n",
       "2019-10-03  33.690254  38.341235  34.809658  29.657188  ...  7.866346   \n",
       "2019-10-04  49.098145  44.835977  43.202534  41.142932  ...  2.602429   \n",
       "\n",
       "                 5319       5320       5321       5322       5323      5324  \\\n",
       "2019-09-30   6.648709   6.888054  15.446540   6.128881   71.70889  2.563082   \n",
       "2019-10-01   9.741949  19.012670  36.024790  13.535630   46.60602  3.803456   \n",
       "2019-10-02  23.461110  33.428940  11.227320  32.616840   36.36791  3.497346   \n",
       "2019-10-03  47.601430  16.523700   1.502065  15.735460  147.13415  1.472838   \n",
       "2019-10-04   5.945044  12.129320   1.327380   7.761159   21.69153  2.312886   \n",
       "\n",
       "                5325        5326       5327  \n",
       "2019-09-30  0.654434         NaN   8.045380  \n",
       "2019-10-01  0.691928    1.578971   1.534836  \n",
       "2019-10-02  0.254489   34.839195  29.089000  \n",
       "2019-10-03  0.353094  164.752345  13.292680  \n",
       "2019-10-04  0.734246   54.006198  20.799630  \n",
       "\n",
       "[5 rows x 5328 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pred = df_pred.dropna(axis=0, how='all')\n",
    "index = df_pred.index\n",
    "df_pred.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "wccZaAz6bpY9"
   },
   "source": [
    "## 將PMF=NAN進行補值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kTra-kQkbpY-",
    "outputId": "807cbc94-f76c-4582-cca4-759eeb27244f"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <span>R/rpy2 DataFrame (33 x 5328)</span>\n",
       "    <table>\n",
       "      <thead>\n",
       "        <tr>\n",
       "        \n",
       "          <th>X0</th>\n",
       "        \n",
       "          <th>X1</th>\n",
       "        \n",
       "          <th>X2</th>\n",
       "        \n",
       "          <th>...</th>\n",
       "        \n",
       "          <th>X5325</th>\n",
       "        \n",
       "          <th>X5326</th>\n",
       "        \n",
       "          <th>X5327</th>\n",
       "        \n",
       "        </tr>\n",
       "      </thead>\n",
       "      <tbody>\n",
       "      \n",
       "      <tr>\n",
       "      \n",
       "      <td>\n",
       "        10.384020\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        12.222376\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        5.971256\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        0.654434\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "             nan\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        8.045380\n",
       "      </td>\n",
       "      \n",
       "      </tr>\n",
       "      \n",
       "      <tr>\n",
       "      \n",
       "      <td>\n",
       "        2.303247\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        1.167996\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        3.119260\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        0.691928\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        1.578971\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        1.534836\n",
       "      </td>\n",
       "      \n",
       "      </tr>\n",
       "      \n",
       "      <tr>\n",
       "      \n",
       "      <td>\n",
       "        18.529622\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        16.354433\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        18.980730\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        0.254489\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        34.839195\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        29.089000\n",
       "      </td>\n",
       "      \n",
       "      </tr>\n",
       "      \n",
       "      <tr>\n",
       "      \n",
       "      <td>\n",
       "        25.217936\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        27.691776\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        26.745528\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        0.353094\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        164.752345\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        13.292680\n",
       "      </td>\n",
       "      \n",
       "      </tr>\n",
       "      \n",
       "      <tr>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      </tr>\n",
       "      \n",
       "      <tr>\n",
       "      \n",
       "      <td>\n",
       "        5.632875\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        4.230715\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        0.269621\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        1.491971\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        2.582207\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        9.006240\n",
       "      </td>\n",
       "      \n",
       "      </tr>\n",
       "      \n",
       "      <tr>\n",
       "      \n",
       "      <td>\n",
       "        1.289931\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        19.258952\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        0.432383\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        0.174710\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        0.259247\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        4.808150\n",
       "      </td>\n",
       "      \n",
       "      </tr>\n",
       "      \n",
       "      <tr>\n",
       "      \n",
       "      <td>\n",
       "        1.264438\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        29.788514\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        0.255462\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        0.535734\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        0.256727\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        5.043830\n",
       "      </td>\n",
       "      \n",
       "      </tr>\n",
       "      \n",
       "      <tr>\n",
       "      \n",
       "      <td>\n",
       "        2.259185\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        19.170722\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        0.316616\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        ...\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        0.360062\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        0.444094\n",
       "      </td>\n",
       "      \n",
       "      <td>\n",
       "        17.873880\n",
       "      </td>\n",
       "      \n",
       "      </tr>\n",
       "      \n",
       "      </tbody>\n",
       "    </table>\n",
       "    "
      ],
      "text/plain": [
       "R object with classes: ('data.frame',) mapped to:\n",
       "<DataFrame - Python:0x000001C484B1AD48 / R:0x000001C48A7086B0>\n",
       "[FloatVector, FloatVector, FloatVector, FloatVector, ..., FloatVector, FloatVector, FloatVector, FloatVector]\n",
       "  X0: <class 'rpy2.robjects.vectors.FloatVector'>\n",
       "  R object with classes: ('numeric',) mapped to:\n",
       "<FloatVector - Python:0x000001C484B1AC48 / R:0x000001C4E635F468>\n",
       "[10.384020, 2.303247, 18.529622, 25.217936, ..., 5.632875, 1.289931, 1.264438, 2.259185]\n",
       "  X1: <class 'rpy2.robjects.vectors.FloatVector'>\n",
       "  R object with classes: ('numeric',) mapped to:\n",
       "<FloatVector - Python:0x000001C484B1AE48 / R:0x000001C4E635FA18>\n",
       "[12.222376, 1.167996, 16.354433, 27.691776, ..., 4.230715, 19.258952, 29.788514, 19.170722]\n",
       "  X2: <class 'rpy2.robjects.vectors.FloatVector'>\n",
       "  R object with classes: ('numeric',) mapped to:\n",
       "<FloatVector - Python:0x000001C484B1AF48 / R:0x000001C4DED11328>\n",
       "[5.971256, 3.119260, 18.980730, 26.745528, ..., 0.269621, 0.432383, 0.255462, 0.316616]\n",
       "  X3: <class 'rpy2.robjects.vectors.FloatVector'>\n",
       "  R object with classes: ('numeric',) mapped to:\n",
       "<FloatVector - Python:0x000001C484B1A948 / R:0x000001C4DED116A0>\n",
       "[11.441598, 1.364694, 14.328082, 24.618408, ..., 2.059012, 14.901676, 22.720576, 20.142869]\n",
       "...\n",
       "  X5: <class 'rpy2.robjects.vectors.FloatVector'>\n",
       "  R object with classes: ('numeric',) mapped to:\n",
       "<FloatVector - Python:0x000001C484B11708 / R:0x000001C4E79F5698>\n",
       "[2.563082, 3.803456, 3.497346, 1.472838, ..., 16.586438, 18.146720, 26.542503, 14.749970]\n",
       "  X6: <class 'rpy2.robjects.vectors.FloatVector'>\n",
       "  R object with classes: ('numeric',) mapped to:\n",
       "<FloatVector - Python:0x000001C484B67088 / R:0x000001C4E79F7998>\n",
       "[0.654434, 0.691928, 0.254489, 0.353094, ..., 1.491971, 0.174710, 0.535734, 0.360062]\n",
       "  X7: <class 'rpy2.robjects.vectors.FloatVector'>\n",
       "  R object with classes: ('numeric',) mapped to:\n",
       "<FloatVector - Python:0x000001C484B67148 / R:0x000001C4E7876488>\n",
       "[     nan, 1.578971, 34.839195, 164.752345, ..., 2.582207, 0.259247, 0.256727, 0.444094]\n",
       "  X8: <class 'rpy2.robjects.vectors.FloatVector'>\n",
       "  R object with classes: ('numeric',) mapped to:\n",
       "<FloatVector - Python:0x000001C484B67248 / R:0x000001C4E7878788>\n",
       "[8.045380, 1.534836, 29.089000, 13.292680, ..., 9.006240, 4.808150, 5.043830, 17.873880]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rpy2_new_prediction = pandas2ri.py2ri(df_pred)\n",
    "robjects.r.assign('rpy2_new_prediction', rpy2_new_prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "n2AZNAGKbpZA",
    "outputId": "d43ff54d-21fb-4964-f0ba-0c35e05fe6a0"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X0</th>\n",
       "      <th>X1</th>\n",
       "      <th>X2</th>\n",
       "      <th>X3</th>\n",
       "      <th>X4</th>\n",
       "      <th>X5</th>\n",
       "      <th>X6</th>\n",
       "      <th>X7</th>\n",
       "      <th>X8</th>\n",
       "      <th>X9</th>\n",
       "      <th>...</th>\n",
       "      <th>X5318</th>\n",
       "      <th>X5319</th>\n",
       "      <th>X5320</th>\n",
       "      <th>X5321</th>\n",
       "      <th>X5322</th>\n",
       "      <th>X5323</th>\n",
       "      <th>X5324</th>\n",
       "      <th>X5325</th>\n",
       "      <th>X5326</th>\n",
       "      <th>X5327</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>10.384020</td>\n",
       "      <td>12.222376</td>\n",
       "      <td>5.971256</td>\n",
       "      <td>11.441598</td>\n",
       "      <td>10.245286</td>\n",
       "      <td>14.770390</td>\n",
       "      <td>14.752220</td>\n",
       "      <td>16.288942</td>\n",
       "      <td>15.054422</td>\n",
       "      <td>9.530121</td>\n",
       "      <td>...</td>\n",
       "      <td>6.551325</td>\n",
       "      <td>6.648709</td>\n",
       "      <td>6.888054</td>\n",
       "      <td>15.446540</td>\n",
       "      <td>6.128881</td>\n",
       "      <td>71.70889</td>\n",
       "      <td>2.563082</td>\n",
       "      <td>0.654434</td>\n",
       "      <td>16.030363</td>\n",
       "      <td>8.045380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2.303247</td>\n",
       "      <td>1.167996</td>\n",
       "      <td>3.119260</td>\n",
       "      <td>1.364694</td>\n",
       "      <td>2.355424</td>\n",
       "      <td>3.042923</td>\n",
       "      <td>3.324206</td>\n",
       "      <td>9.170274</td>\n",
       "      <td>2.758917</td>\n",
       "      <td>1.507425</td>\n",
       "      <td>...</td>\n",
       "      <td>8.073354</td>\n",
       "      <td>9.741949</td>\n",
       "      <td>19.012670</td>\n",
       "      <td>36.024790</td>\n",
       "      <td>13.535630</td>\n",
       "      <td>46.60602</td>\n",
       "      <td>3.803456</td>\n",
       "      <td>0.691928</td>\n",
       "      <td>1.578971</td>\n",
       "      <td>1.534836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>18.529622</td>\n",
       "      <td>16.354433</td>\n",
       "      <td>18.980730</td>\n",
       "      <td>14.328082</td>\n",
       "      <td>18.858808</td>\n",
       "      <td>20.995500</td>\n",
       "      <td>22.920832</td>\n",
       "      <td>18.492727</td>\n",
       "      <td>28.264308</td>\n",
       "      <td>17.527233</td>\n",
       "      <td>...</td>\n",
       "      <td>8.417409</td>\n",
       "      <td>23.461110</td>\n",
       "      <td>33.428940</td>\n",
       "      <td>11.227320</td>\n",
       "      <td>32.616840</td>\n",
       "      <td>36.36791</td>\n",
       "      <td>3.497346</td>\n",
       "      <td>0.254489</td>\n",
       "      <td>34.839195</td>\n",
       "      <td>29.089000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>25.217936</td>\n",
       "      <td>27.691776</td>\n",
       "      <td>26.745528</td>\n",
       "      <td>24.618408</td>\n",
       "      <td>33.900899</td>\n",
       "      <td>35.589010</td>\n",
       "      <td>33.690254</td>\n",
       "      <td>38.341235</td>\n",
       "      <td>34.809658</td>\n",
       "      <td>29.657188</td>\n",
       "      <td>...</td>\n",
       "      <td>7.866346</td>\n",
       "      <td>47.601430</td>\n",
       "      <td>16.523700</td>\n",
       "      <td>1.502065</td>\n",
       "      <td>15.735460</td>\n",
       "      <td>147.13415</td>\n",
       "      <td>1.472838</td>\n",
       "      <td>0.353094</td>\n",
       "      <td>164.752345</td>\n",
       "      <td>13.292680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>38.854236</td>\n",
       "      <td>36.755066</td>\n",
       "      <td>35.572574</td>\n",
       "      <td>34.110252</td>\n",
       "      <td>48.683100</td>\n",
       "      <td>52.354600</td>\n",
       "      <td>49.098145</td>\n",
       "      <td>44.835977</td>\n",
       "      <td>43.202534</td>\n",
       "      <td>41.142932</td>\n",
       "      <td>...</td>\n",
       "      <td>2.602429</td>\n",
       "      <td>5.945044</td>\n",
       "      <td>12.129320</td>\n",
       "      <td>1.327380</td>\n",
       "      <td>7.761159</td>\n",
       "      <td>21.69153</td>\n",
       "      <td>2.312886</td>\n",
       "      <td>0.734246</td>\n",
       "      <td>54.006198</td>\n",
       "      <td>20.799630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>17.153744</td>\n",
       "      <td>17.063243</td>\n",
       "      <td>13.142354</td>\n",
       "      <td>15.819230</td>\n",
       "      <td>24.312800</td>\n",
       "      <td>26.722910</td>\n",
       "      <td>25.678790</td>\n",
       "      <td>22.525031</td>\n",
       "      <td>21.538990</td>\n",
       "      <td>15.539159</td>\n",
       "      <td>...</td>\n",
       "      <td>17.842642</td>\n",
       "      <td>9.017360</td>\n",
       "      <td>6.484012</td>\n",
       "      <td>6.350275</td>\n",
       "      <td>6.738180</td>\n",
       "      <td>95.40394</td>\n",
       "      <td>1.535621</td>\n",
       "      <td>0.550834</td>\n",
       "      <td>32.301083</td>\n",
       "      <td>25.989810</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6 rows × 5328 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          X0         X1         X2         X3         X4         X5  \\\n",
       "0  10.384020  12.222376   5.971256  11.441598  10.245286  14.770390   \n",
       "1   2.303247   1.167996   3.119260   1.364694   2.355424   3.042923   \n",
       "2  18.529622  16.354433  18.980730  14.328082  18.858808  20.995500   \n",
       "3  25.217936  27.691776  26.745528  24.618408  33.900899  35.589010   \n",
       "4  38.854236  36.755066  35.572574  34.110252  48.683100  52.354600   \n",
       "5  17.153744  17.063243  13.142354  15.819230  24.312800  26.722910   \n",
       "\n",
       "          X6         X7         X8         X9  ...      X5318      X5319  \\\n",
       "0  14.752220  16.288942  15.054422   9.530121  ...   6.551325   6.648709   \n",
       "1   3.324206   9.170274   2.758917   1.507425  ...   8.073354   9.741949   \n",
       "2  22.920832  18.492727  28.264308  17.527233  ...   8.417409  23.461110   \n",
       "3  33.690254  38.341235  34.809658  29.657188  ...   7.866346  47.601430   \n",
       "4  49.098145  44.835977  43.202534  41.142932  ...   2.602429   5.945044   \n",
       "5  25.678790  22.525031  21.538990  15.539159  ...  17.842642   9.017360   \n",
       "\n",
       "       X5320      X5321      X5322      X5323     X5324     X5325       X5326  \\\n",
       "0   6.888054  15.446540   6.128881   71.70889  2.563082  0.654434   16.030363   \n",
       "1  19.012670  36.024790  13.535630   46.60602  3.803456  0.691928    1.578971   \n",
       "2  33.428940  11.227320  32.616840   36.36791  3.497346  0.254489   34.839195   \n",
       "3  16.523700   1.502065  15.735460  147.13415  1.472838  0.353094  164.752345   \n",
       "4  12.129320   1.327380   7.761159   21.69153  2.312886  0.734246   54.006198   \n",
       "5   6.484012   6.350275   6.738180   95.40394  1.535621  0.550834   32.301083   \n",
       "\n",
       "       X5327  \n",
       "0   8.045380  \n",
       "1   1.534836  \n",
       "2  29.089000  \n",
       "3  13.292680  \n",
       "4  20.799630  \n",
       "5  25.989810  \n",
       "\n",
       "[6 rows x 5328 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r('''\n",
    "rpy2_new_prediction[is.na(rpy2_new_prediction)]<- NaN\n",
    "y <- data.matrix(rpy2_new_prediction)\n",
    "Ya <- dineof(y)$Xa\n",
    "new_prediction<-data.frame(Ya)\n",
    "head(new_prediction)\n",
    "''')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kCXFWIz1bpZC",
    "outputId": "72f7227a-3381-4f19-8442-a4d205bdbdeb"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X0</th>\n",
       "      <th>X1</th>\n",
       "      <th>X2</th>\n",
       "      <th>X3</th>\n",
       "      <th>X4</th>\n",
       "      <th>X5</th>\n",
       "      <th>X6</th>\n",
       "      <th>X7</th>\n",
       "      <th>X8</th>\n",
       "      <th>X9</th>\n",
       "      <th>...</th>\n",
       "      <th>X5318</th>\n",
       "      <th>X5319</th>\n",
       "      <th>X5320</th>\n",
       "      <th>X5321</th>\n",
       "      <th>X5322</th>\n",
       "      <th>X5323</th>\n",
       "      <th>X5324</th>\n",
       "      <th>X5325</th>\n",
       "      <th>X5326</th>\n",
       "      <th>X5327</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2019-09-30</td>\n",
       "      <td>10.384020</td>\n",
       "      <td>12.222376</td>\n",
       "      <td>5.971256</td>\n",
       "      <td>11.441598</td>\n",
       "      <td>10.245286</td>\n",
       "      <td>14.770390</td>\n",
       "      <td>14.752220</td>\n",
       "      <td>16.288942</td>\n",
       "      <td>15.054422</td>\n",
       "      <td>9.530121</td>\n",
       "      <td>...</td>\n",
       "      <td>6.551325</td>\n",
       "      <td>6.648709</td>\n",
       "      <td>6.888054</td>\n",
       "      <td>15.446540</td>\n",
       "      <td>6.128881</td>\n",
       "      <td>71.70889</td>\n",
       "      <td>2.563082</td>\n",
       "      <td>0.654434</td>\n",
       "      <td>16.030363</td>\n",
       "      <td>8.045380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-01</td>\n",
       "      <td>2.303247</td>\n",
       "      <td>1.167996</td>\n",
       "      <td>3.119260</td>\n",
       "      <td>1.364694</td>\n",
       "      <td>2.355424</td>\n",
       "      <td>3.042923</td>\n",
       "      <td>3.324206</td>\n",
       "      <td>9.170274</td>\n",
       "      <td>2.758917</td>\n",
       "      <td>1.507425</td>\n",
       "      <td>...</td>\n",
       "      <td>8.073354</td>\n",
       "      <td>9.741949</td>\n",
       "      <td>19.012670</td>\n",
       "      <td>36.024790</td>\n",
       "      <td>13.535630</td>\n",
       "      <td>46.60602</td>\n",
       "      <td>3.803456</td>\n",
       "      <td>0.691928</td>\n",
       "      <td>1.578971</td>\n",
       "      <td>1.534836</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-02</td>\n",
       "      <td>18.529622</td>\n",
       "      <td>16.354433</td>\n",
       "      <td>18.980730</td>\n",
       "      <td>14.328082</td>\n",
       "      <td>18.858808</td>\n",
       "      <td>20.995500</td>\n",
       "      <td>22.920832</td>\n",
       "      <td>18.492727</td>\n",
       "      <td>28.264308</td>\n",
       "      <td>17.527233</td>\n",
       "      <td>...</td>\n",
       "      <td>8.417409</td>\n",
       "      <td>23.461110</td>\n",
       "      <td>33.428940</td>\n",
       "      <td>11.227320</td>\n",
       "      <td>32.616840</td>\n",
       "      <td>36.36791</td>\n",
       "      <td>3.497346</td>\n",
       "      <td>0.254489</td>\n",
       "      <td>34.839195</td>\n",
       "      <td>29.089000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-03</td>\n",
       "      <td>25.217936</td>\n",
       "      <td>27.691776</td>\n",
       "      <td>26.745528</td>\n",
       "      <td>24.618408</td>\n",
       "      <td>33.900899</td>\n",
       "      <td>35.589010</td>\n",
       "      <td>33.690254</td>\n",
       "      <td>38.341235</td>\n",
       "      <td>34.809658</td>\n",
       "      <td>29.657188</td>\n",
       "      <td>...</td>\n",
       "      <td>7.866346</td>\n",
       "      <td>47.601430</td>\n",
       "      <td>16.523700</td>\n",
       "      <td>1.502065</td>\n",
       "      <td>15.735460</td>\n",
       "      <td>147.13415</td>\n",
       "      <td>1.472838</td>\n",
       "      <td>0.353094</td>\n",
       "      <td>164.752345</td>\n",
       "      <td>13.292680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-04</td>\n",
       "      <td>38.854236</td>\n",
       "      <td>36.755066</td>\n",
       "      <td>35.572574</td>\n",
       "      <td>34.110252</td>\n",
       "      <td>48.683100</td>\n",
       "      <td>52.354600</td>\n",
       "      <td>49.098145</td>\n",
       "      <td>44.835977</td>\n",
       "      <td>43.202534</td>\n",
       "      <td>41.142932</td>\n",
       "      <td>...</td>\n",
       "      <td>2.602429</td>\n",
       "      <td>5.945044</td>\n",
       "      <td>12.129320</td>\n",
       "      <td>1.327380</td>\n",
       "      <td>7.761159</td>\n",
       "      <td>21.69153</td>\n",
       "      <td>2.312886</td>\n",
       "      <td>0.734246</td>\n",
       "      <td>54.006198</td>\n",
       "      <td>20.799630</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 5328 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   X0         X1         X2         X3         X4         X5  \\\n",
       "2019-09-30  10.384020  12.222376   5.971256  11.441598  10.245286  14.770390   \n",
       "2019-10-01   2.303247   1.167996   3.119260   1.364694   2.355424   3.042923   \n",
       "2019-10-02  18.529622  16.354433  18.980730  14.328082  18.858808  20.995500   \n",
       "2019-10-03  25.217936  27.691776  26.745528  24.618408  33.900899  35.589010   \n",
       "2019-10-04  38.854236  36.755066  35.572574  34.110252  48.683100  52.354600   \n",
       "\n",
       "                   X6         X7         X8         X9  ...     X5318  \\\n",
       "2019-09-30  14.752220  16.288942  15.054422   9.530121  ...  6.551325   \n",
       "2019-10-01   3.324206   9.170274   2.758917   1.507425  ...  8.073354   \n",
       "2019-10-02  22.920832  18.492727  28.264308  17.527233  ...  8.417409   \n",
       "2019-10-03  33.690254  38.341235  34.809658  29.657188  ...  7.866346   \n",
       "2019-10-04  49.098145  44.835977  43.202534  41.142932  ...  2.602429   \n",
       "\n",
       "                X5319      X5320      X5321      X5322      X5323     X5324  \\\n",
       "2019-09-30   6.648709   6.888054  15.446540   6.128881   71.70889  2.563082   \n",
       "2019-10-01   9.741949  19.012670  36.024790  13.535630   46.60602  3.803456   \n",
       "2019-10-02  23.461110  33.428940  11.227320  32.616840   36.36791  3.497346   \n",
       "2019-10-03  47.601430  16.523700   1.502065  15.735460  147.13415  1.472838   \n",
       "2019-10-04   5.945044  12.129320   1.327380   7.761159   21.69153  2.312886   \n",
       "\n",
       "               X5325       X5326      X5327  \n",
       "2019-09-30  0.654434   16.030363   8.045380  \n",
       "2019-10-01  0.691928    1.578971   1.534836  \n",
       "2019-10-02  0.254489   34.839195  29.089000  \n",
       "2019-10-03  0.353094  164.752345  13.292680  \n",
       "2019-10-04  0.734246   54.006198  20.799630  \n",
       "\n",
       "[5 rows x 5328 columns]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r.data('new_prediction')\n",
    "new_prediction = pd.DataFrame(r['new_prediction'])\n",
    "new_prediction.index = index\n",
    "new_prediction.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>X0</th>\n",
       "      <th>X1</th>\n",
       "      <th>X2</th>\n",
       "      <th>X3</th>\n",
       "      <th>X4</th>\n",
       "      <th>X5</th>\n",
       "      <th>X6</th>\n",
       "      <th>X7</th>\n",
       "      <th>X8</th>\n",
       "      <th>X9</th>\n",
       "      <th>...</th>\n",
       "      <th>X5318</th>\n",
       "      <th>X5319</th>\n",
       "      <th>X5320</th>\n",
       "      <th>X5321</th>\n",
       "      <th>X5322</th>\n",
       "      <th>X5323</th>\n",
       "      <th>X5324</th>\n",
       "      <th>X5325</th>\n",
       "      <th>X5326</th>\n",
       "      <th>X5327</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2019-10-28</td>\n",
       "      <td>13.427509</td>\n",
       "      <td>13.446619</td>\n",
       "      <td>12.398126</td>\n",
       "      <td>14.050098</td>\n",
       "      <td>16.355950</td>\n",
       "      <td>20.087620</td>\n",
       "      <td>16.750780</td>\n",
       "      <td>18.338449</td>\n",
       "      <td>19.555760</td>\n",
       "      <td>16.339273</td>\n",
       "      <td>...</td>\n",
       "      <td>8.265961</td>\n",
       "      <td>7.216942</td>\n",
       "      <td>7.790610</td>\n",
       "      <td>14.271270</td>\n",
       "      <td>4.566690</td>\n",
       "      <td>119.616840</td>\n",
       "      <td>4.381921</td>\n",
       "      <td>0.757728</td>\n",
       "      <td>11.389323</td>\n",
       "      <td>5.244024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-29</td>\n",
       "      <td>5.632875</td>\n",
       "      <td>4.230715</td>\n",
       "      <td>0.269621</td>\n",
       "      <td>2.059012</td>\n",
       "      <td>11.498530</td>\n",
       "      <td>11.264280</td>\n",
       "      <td>6.891853</td>\n",
       "      <td>10.196930</td>\n",
       "      <td>4.680984</td>\n",
       "      <td>2.894081</td>\n",
       "      <td>...</td>\n",
       "      <td>3.410336</td>\n",
       "      <td>7.111055</td>\n",
       "      <td>10.344857</td>\n",
       "      <td>65.649088</td>\n",
       "      <td>9.054913</td>\n",
       "      <td>26.173115</td>\n",
       "      <td>16.586438</td>\n",
       "      <td>1.491971</td>\n",
       "      <td>2.582207</td>\n",
       "      <td>9.006240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-30</td>\n",
       "      <td>1.289931</td>\n",
       "      <td>19.258952</td>\n",
       "      <td>0.432383</td>\n",
       "      <td>14.901676</td>\n",
       "      <td>14.629965</td>\n",
       "      <td>15.920009</td>\n",
       "      <td>17.553282</td>\n",
       "      <td>19.671755</td>\n",
       "      <td>11.698523</td>\n",
       "      <td>16.916361</td>\n",
       "      <td>...</td>\n",
       "      <td>2.414034</td>\n",
       "      <td>6.646315</td>\n",
       "      <td>4.300676</td>\n",
       "      <td>23.151850</td>\n",
       "      <td>3.250110</td>\n",
       "      <td>26.905080</td>\n",
       "      <td>18.146720</td>\n",
       "      <td>0.174710</td>\n",
       "      <td>0.259247</td>\n",
       "      <td>4.808150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-31</td>\n",
       "      <td>1.264438</td>\n",
       "      <td>29.788514</td>\n",
       "      <td>0.255462</td>\n",
       "      <td>22.720576</td>\n",
       "      <td>12.146773</td>\n",
       "      <td>22.353600</td>\n",
       "      <td>29.110149</td>\n",
       "      <td>34.981333</td>\n",
       "      <td>34.254231</td>\n",
       "      <td>25.641742</td>\n",
       "      <td>...</td>\n",
       "      <td>3.009589</td>\n",
       "      <td>3.709717</td>\n",
       "      <td>7.282560</td>\n",
       "      <td>23.191280</td>\n",
       "      <td>3.551979</td>\n",
       "      <td>36.680530</td>\n",
       "      <td>26.542503</td>\n",
       "      <td>0.535734</td>\n",
       "      <td>0.256727</td>\n",
       "      <td>5.043830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-11-01</td>\n",
       "      <td>2.259185</td>\n",
       "      <td>19.170722</td>\n",
       "      <td>0.316616</td>\n",
       "      <td>20.142869</td>\n",
       "      <td>9.882625</td>\n",
       "      <td>20.470040</td>\n",
       "      <td>20.447631</td>\n",
       "      <td>22.565321</td>\n",
       "      <td>22.329302</td>\n",
       "      <td>19.088418</td>\n",
       "      <td>...</td>\n",
       "      <td>3.513030</td>\n",
       "      <td>6.379925</td>\n",
       "      <td>11.479990</td>\n",
       "      <td>27.022449</td>\n",
       "      <td>3.002634</td>\n",
       "      <td>51.187390</td>\n",
       "      <td>14.749970</td>\n",
       "      <td>0.360062</td>\n",
       "      <td>0.444094</td>\n",
       "      <td>17.873880</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 5328 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   X0         X1         X2         X3         X4         X5  \\\n",
       "2019-10-28  13.427509  13.446619  12.398126  14.050098  16.355950  20.087620   \n",
       "2019-10-29   5.632875   4.230715   0.269621   2.059012  11.498530  11.264280   \n",
       "2019-10-30   1.289931  19.258952   0.432383  14.901676  14.629965  15.920009   \n",
       "2019-10-31   1.264438  29.788514   0.255462  22.720576  12.146773  22.353600   \n",
       "2019-11-01   2.259185  19.170722   0.316616  20.142869   9.882625  20.470040   \n",
       "\n",
       "                   X6         X7         X8         X9  ...     X5318  \\\n",
       "2019-10-28  16.750780  18.338449  19.555760  16.339273  ...  8.265961   \n",
       "2019-10-29   6.891853  10.196930   4.680984   2.894081  ...  3.410336   \n",
       "2019-10-30  17.553282  19.671755  11.698523  16.916361  ...  2.414034   \n",
       "2019-10-31  29.110149  34.981333  34.254231  25.641742  ...  3.009589   \n",
       "2019-11-01  20.447631  22.565321  22.329302  19.088418  ...  3.513030   \n",
       "\n",
       "               X5319      X5320      X5321     X5322       X5323      X5324  \\\n",
       "2019-10-28  7.216942   7.790610  14.271270  4.566690  119.616840   4.381921   \n",
       "2019-10-29  7.111055  10.344857  65.649088  9.054913   26.173115  16.586438   \n",
       "2019-10-30  6.646315   4.300676  23.151850  3.250110   26.905080  18.146720   \n",
       "2019-10-31  3.709717   7.282560  23.191280  3.551979   36.680530  26.542503   \n",
       "2019-11-01  6.379925  11.479990  27.022449  3.002634   51.187390  14.749970   \n",
       "\n",
       "               X5325      X5326      X5327  \n",
       "2019-10-28  0.757728  11.389323   5.244024  \n",
       "2019-10-29  1.491971   2.582207   9.006240  \n",
       "2019-10-30  0.174710   0.259247   4.808150  \n",
       "2019-10-31  0.535734   0.256727   5.043830  \n",
       "2019-11-01  0.360062   0.444094  17.873880  \n",
       "\n",
       "[5 rows x 5328 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_prediction.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "SPA2suQibpZE"
   },
   "source": [
    "## Prediction 完成"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "yVzuKpfjbpZE"
   },
   "source": [
    "## 將 Observation 對齊 Prediction 日期"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wV3NYlqmbpZI",
    "outputId": "ba4c1e27-d8f0-4561-8247-71bcb3d42433"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\User\\Anaconda3\\envs\\tf2_rpy2\\lib\\site-packages\\ipykernel_launcher.py:2: FutureWarning: The join_axes-keyword is deprecated. Use .reindex or .reindex_like on the result to achieve the same functionality.\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>5318</th>\n",
       "      <th>5319</th>\n",
       "      <th>5320</th>\n",
       "      <th>5321</th>\n",
       "      <th>5322</th>\n",
       "      <th>5323</th>\n",
       "      <th>5324</th>\n",
       "      <th>5325</th>\n",
       "      <th>5326</th>\n",
       "      <th>5327</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2019-10-28</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>...</td>\n",
       "      <td>12.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-29</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>...</td>\n",
       "      <td>25.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>27.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-30</td>\n",
       "      <td>19.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-31</td>\n",
       "      <td>27.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-11-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 5328 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            0     1     2     3     4     5     6     7     8     9     ...  \\\n",
       "2019-10-28   8.0   9.0   8.0   5.0  11.0  10.0  10.0  14.0  15.0  12.0  ...   \n",
       "2019-10-29   6.0   6.0   3.0   3.0   4.0   2.0   5.0   8.0   5.0   6.0  ...   \n",
       "2019-10-30  19.0  19.0  24.0  15.0  18.0   7.0  12.0  12.0  21.0  18.0  ...   \n",
       "2019-10-31  27.0  26.0  40.0  24.0  29.0  32.0  29.0  32.0  34.0  27.0  ...   \n",
       "2019-11-01   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "\n",
       "            5318  5319  5320  5321  5322  5323  5324  5325  5326  5327  \n",
       "2019-10-28  12.0  42.0  29.0  45.0  25.0  61.0  60.0   7.0  31.0  24.0  \n",
       "2019-10-29  25.0  29.0  21.0  32.0  20.0  37.0  43.0   4.0  19.0  27.0  \n",
       "2019-10-30   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "2019-10-31   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "2019-11-01   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "\n",
       "[5 rows x 5328 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_prediction.index = (new_prediction.index).astype(str)\n",
    "observation_z = pd.concat([observation_z, new_prediction], axis=1, join_axes=[new_prediction.index]).iloc[:, :time_step*station_num]\n",
    "observation_z.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\User\\Anaconda3\\envs\\tf2_rpy2\\lib\\site-packages\\ipykernel_launcher.py:2: FutureWarning: The join_axes-keyword is deprecated. Use .reindex or .reindex_like on the result to achieve the same functionality.\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>5318</th>\n",
       "      <th>5319</th>\n",
       "      <th>5320</th>\n",
       "      <th>5321</th>\n",
       "      <th>5322</th>\n",
       "      <th>5323</th>\n",
       "      <th>5324</th>\n",
       "      <th>5325</th>\n",
       "      <th>5326</th>\n",
       "      <th>5327</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>2019-10-28</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>...</td>\n",
       "      <td>12.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-29</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>...</td>\n",
       "      <td>25.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>37.0</td>\n",
       "      <td>43.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>27.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-30</td>\n",
       "      <td>19.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-10-31</td>\n",
       "      <td>27.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2019-11-01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 5328 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            0     1     2     3     4     5     6     7     8     9     ...  \\\n",
       "2019-10-28   8.0   9.0   8.0   5.0  11.0  10.0  10.0  14.0  15.0  12.0  ...   \n",
       "2019-10-29   6.0   6.0   3.0   3.0   4.0   2.0   5.0   8.0   5.0   6.0  ...   \n",
       "2019-10-30  19.0  19.0  24.0  15.0  18.0   7.0  12.0  12.0  21.0  18.0  ...   \n",
       "2019-10-31  27.0  26.0  40.0  24.0  29.0  32.0  29.0  32.0  34.0  27.0  ...   \n",
       "2019-11-01   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "\n",
       "            5318  5319  5320  5321  5322  5323  5324  5325  5326  5327  \n",
       "2019-10-28  12.0  42.0  29.0  45.0  25.0  61.0  60.0   7.0  31.0  24.0  \n",
       "2019-10-29  25.0  29.0  21.0  32.0  20.0  37.0  43.0   4.0  19.0  27.0  \n",
       "2019-10-30   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "2019-10-31   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "2019-11-01   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "\n",
       "[5 rows x 5328 columns]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_prediction.index = (new_prediction.index).astype(str)\n",
    "observation_y = pd.concat([observation_y, new_prediction], axis=1, join_axes=[new_prediction.index]).iloc[:, :time_step * station_num]\n",
    "observation_y.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "PsxQ2DsibpZK"
   },
   "source": [
    "## Observation 完成"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "zkHnAGC8bpZK"
   },
   "source": [
    "# ----------------------------------------------------------------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "J2QnP_l1bpZL"
   },
   "source": [
    "## Model 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 33
    },
    "colab_type": "code",
    "id": "ojyXV0IQbpZL",
    "outputId": "7308d1b8-1057-472e-8902-a59255b3fdcd"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.keras import layers\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.client.session.Session at 0x1c48c16e0f0>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config = tf.compat.v1.ConfigProto()\n",
    "config.gpu_options.allow_growth = True\n",
    "sess = tf.compat.v1.Session(config=config)\n",
    "config = tf.compat.v1.ConfigProto()\n",
    "config.gpu_options.per_process_gpu_memory_fraction = 0.95\n",
    "tf.compat.v1.Session(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 67
    },
    "colab_type": "code",
    "id": "06fm49dPbpZM",
    "outputId": "edccef51-bb53-47cd-86f7-e64952191355"
   },
   "outputs": [],
   "source": [
    "XData = np.array(new_prediction)[1:,:].astype('float32')\n",
    "ZData = np.array(observation_z)[:-1,0:station_num*24].astype('float32')  #往前看24小時\n",
    "YData = np.array(observation_y)[1:,:].astype('float32')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "vsjwka53ni8M"
   },
   "source": [
    "## 比較校正後的結果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MpVvFvsrbpZf"
   },
   "outputs": [],
   "source": [
    "new_model = tf.keras.models.load_model('model4_mne_softplus_1730_72.h5',compile=False)\n",
    "y_hat = new_model([XData, ZData], training = False).numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "m--257l7bpZg"
   },
   "source": [
    "## 將資料整理回原格式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kIX3q5Jani8T",
    "outputId": "b6b30982-e690-46ee-fb52-86ea19efbc8c"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\User\\Anaconda3\\envs\\tf2_rpy2\\lib\\site-packages\\ipykernel_launcher.py:23: DeprecationWarning: Non-string object detected for the array ordering. Please pass in 'C', 'F', 'A', or 'K' instead\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>expdate</th>\n",
       "      <th>stationID</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>hour</th>\n",
       "      <th>tau</th>\n",
       "      <th>PMf_ori</th>\n",
       "      <th>PMf_corr</th>\n",
       "      <th>observation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2019-09-30</td>\n",
       "      <td>EPA001</td>\n",
       "      <td>2019</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2.303247</td>\n",
       "      <td>4.804333</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>2019-09-30</td>\n",
       "      <td>EPA002</td>\n",
       "      <td>2019</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.167996</td>\n",
       "      <td>10.514338</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2019-09-30</td>\n",
       "      <td>EPA003</td>\n",
       "      <td>2019</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3.119260</td>\n",
       "      <td>4.602078</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2019-09-30</td>\n",
       "      <td>EPA004</td>\n",
       "      <td>2019</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.364694</td>\n",
       "      <td>8.056275</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>2019-09-30</td>\n",
       "      <td>EPA005</td>\n",
       "      <td>2019</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2.355424</td>\n",
       "      <td>7.557329</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      expdate stationID  year  month  day  hour  tau   PMf_ori   PMf_corr  \\\n",
       "0  2019-09-30    EPA001  2019     10    1     1    1  2.303247   4.804333   \n",
       "1  2019-09-30    EPA002  2019     10    1     1    1  1.167996  10.514338   \n",
       "2  2019-09-30    EPA003  2019     10    1     1    1  3.119260   4.602078   \n",
       "3  2019-09-30    EPA004  2019     10    1     1    1  1.364694   8.056275   \n",
       "4  2019-09-30    EPA005  2019     10    1     1    1  2.355424   7.557329   \n",
       "\n",
       "   observation  \n",
       "0          6.0  \n",
       "1          5.0  \n",
       "2          8.0  \n",
       "3          1.0  \n",
       "4          5.0  "
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stations = np.unique(df_prediction['stationID'])\n",
    "correction = list(y_hat.reshape(-1,1).flatten())\n",
    "Pred = list(XData.flatten())\n",
    "expdate = (pd.to_datetime(new_prediction.index)[1:] - timedelta(days=1)).astype(str).repeat(time_step*station_num)\n",
    "stationID = list(stations) * time_step *len(XData)\n",
    "tau = list((np.arange(time_step) + 1).repeat(len(stations))) * len(XData)\n",
    "hour = np.array(tau) % 24\n",
    "prediction = { \n",
    "             'expdate':expdate,\n",
    "             'stationID':stationID,\n",
    "             'tau':tau,\n",
    "             'hour':hour,\n",
    "             \"PMf_ori\": Pred,\n",
    "             \"PMf_corr\": correction}\n",
    "prediction_df = pd.DataFrame(prediction)\n",
    "prediction_df['day'] = (pd.to_datetime(prediction_df['expdate']) + \n",
    "                        pd.TimedeltaIndex(23 + prediction_df['tau'], unit='h')).dt.day\n",
    "prediction_df['month'] = (pd.to_datetime(prediction_df['expdate']) + \n",
    "                        pd.TimedeltaIndex(23 + prediction_df['tau'], unit='h')).dt.month\n",
    "prediction_df['year'] = (pd.to_datetime(prediction_df['expdate']) + \n",
    "                        pd.TimedeltaIndex(23 + prediction_df['tau'], unit='h')).dt.year\n",
    "prediction_df = prediction_df[['expdate','stationID','year','month','day','hour','tau','PMf_ori','PMf_corr']]\n",
    "prediction_df['observation'] = np.array(observation_y[1:]).flatten(0)\n",
    "prediction_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>expdate</th>\n",
       "      <th>stationID</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>day</th>\n",
       "      <th>hour</th>\n",
       "      <th>tau</th>\n",
       "      <th>PMf_ori</th>\n",
       "      <th>PMf_corr</th>\n",
       "      <th>observation</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>170491</td>\n",
       "      <td>2019-10-31</td>\n",
       "      <td>EPA071</td>\n",
       "      <td>2019</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>72</td>\n",
       "      <td>51.187389</td>\n",
       "      <td>23.884083</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>170492</td>\n",
       "      <td>2019-10-31</td>\n",
       "      <td>EPA072</td>\n",
       "      <td>2019</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>72</td>\n",
       "      <td>14.749970</td>\n",
       "      <td>20.816330</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>170493</td>\n",
       "      <td>2019-10-31</td>\n",
       "      <td>EPA080</td>\n",
       "      <td>2019</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>72</td>\n",
       "      <td>0.360062</td>\n",
       "      <td>3.954753</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>170494</td>\n",
       "      <td>2019-10-31</td>\n",
       "      <td>EPA102</td>\n",
       "      <td>2019</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>72</td>\n",
       "      <td>0.444094</td>\n",
       "      <td>11.671880</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>170495</td>\n",
       "      <td>2019-10-31</td>\n",
       "      <td>EPA110</td>\n",
       "      <td>2019</td>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>72</td>\n",
       "      <td>17.873880</td>\n",
       "      <td>20.255180</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           expdate stationID  year  month  day  hour  tau    PMf_ori  \\\n",
       "170491  2019-10-31    EPA071  2019     11    3     0   72  51.187389   \n",
       "170492  2019-10-31    EPA072  2019     11    3     0   72  14.749970   \n",
       "170493  2019-10-31    EPA080  2019     11    3     0   72   0.360062   \n",
       "170494  2019-10-31    EPA102  2019     11    3     0   72   0.444094   \n",
       "170495  2019-10-31    EPA110  2019     11    3     0   72  17.873880   \n",
       "\n",
       "         PMf_corr  observation  \n",
       "170491  23.884083          NaN  \n",
       "170492  20.816330          NaN  \n",
       "170493   3.954753          NaN  \n",
       "170494  11.671880          NaN  \n",
       "170495  20.255180          NaN  "
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prediction_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "def error_print(df):\n",
    "    def error(Y_hat, real):\n",
    "        MNE = np.nanmean(abs((Y_hat[real!=0] - real[real!=0])/(real[real!=0]))) \n",
    "        MNB = np.nanmean((Y_hat[real!=0] - real[real!=0])/(real[real!=0]))\n",
    "        return MNE, MNB\n",
    "    df = df.dropna(subset=[\"observation\"])  \n",
    "                      \n",
    "    MNE_corr, MNB_corr = error(df['PMf_corr'], df['observation'])  \n",
    "    print('regression Mean Error = ', np.nanmean(abs(df['PMf_corr']-df['observation'])))\n",
    "    print('regression Mean Normalized Error = ', MNE_corr)\n",
    "    print('regression Mean Normalized Bias = ', MNB_corr)\n",
    "    print('regression correlation coefficient = ', df.PMf_corr.corr(df.observation))\n",
    "    print(' ')                  \n",
    "    MNE_ori, MNB_ori = error(df['PMf_ori'], df['observation'])\n",
    "    print('original Mean  Absolute Error = ', np.nanmean(abs(df['PMf_ori']-df['observation'])))\n",
    "    print('original Mean Normalized Error = ', MNE_ori)\n",
    "    print('original Mean Normalized Bias = ', MNB_ori)\n",
    "    print('original correlation coefficient = ', df.PMf_ori.corr(df.observation))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "regression Mean Error =  11.744259283681062\n",
      "regression Mean Normalized Error =  0.4705529310495956\n",
      "regression Mean Normalized Bias =  -0.3349991388752119\n",
      "regression correlation coefficient =  0.43129006924044305\n",
      " \n",
      "original Mean  Absolute Error =  18.84130871119498\n",
      "original Mean Normalized Error =  0.753791753658697\n",
      "original Mean Normalized Bias =  -0.3755286386094416\n",
      "original correlation coefficient =  0.15558535844284654\n"
     ]
    }
   ],
   "source": [
    "error_print(prediction_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZXgDajvhni8V"
   },
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-65-df285bb8d2d4>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mprediction_df\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0margv\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m'/'\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0msys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0margv\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m6\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "prediction_df.to_csv(sys.argv[5] + '/' + sys.argv[6])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "correction.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}