Correction_time.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Correction_time.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.9"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "YgPItvgqpbfr"
      },
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "#%matplotlib inline\n",
        "import time\n",
        "from datetime import datetime\n",
        "'''\n",
        "import rpy2.robjects as robjects\n",
        "from rpy2.robjects.packages import importr\n",
        "from rpy2.robjects import r, numpy2ri, FloatVector, pandas2ri\n",
        "'''\n",
        "from datetime import timedelta\n",
        "from IPython import get_ipython\n",
        "import sys"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WjFsgdvQbpXy"
      },
      "source": [
        "# Observation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0-cjtFj-bpXz"
      },
      "source": [
        "## 定義所採納的時間"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l0mA1mB4ni6N",
        "outputId": "2eb02d47-d57d-4435-f0ac-cbc697bc7571",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 198
        }
      },
      "source": [
        "'''\n",
        "start = str(sys.argv[1]) + ' 00:00:00' #input('起始日期：(格式:2019-01-01)')\n",
        "end = str(sys.argv[2]) + ' 23:00:00' #input('結束日期：(格式:2019-10-17)')\n",
        "time_step = int(sys.argv[3]) #int(input('預報小時：(格式:96)'))\n",
        "ver = str(sys.argv[4]).split(\",\") #[int(num) for num in input(\"請輸入全部的版本(格式：17 18),以空白鍵區隔：\").split(\",\")] \n",
        "'''"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'\\nstart = str(sys.argv[1]) + \\' 00:00:00\\' #input(\\'起始日期：(格式:2019-01-01)\\')\\nend = str(sys.argv[2]) + \\' 23:00:00\\' #input(\\'結束日期：(格式:2019-10-17)\\')\\ntime_step = int(sys.argv[3]) #int(input(\\'預報小時：(格式:96)\\'))\\nver = str(sys.argv[4]).split(\",\") #[int(num) for num in input(\"請輸入全部的版本(格式：17 18),以空白鍵區隔：\").split(\",\")] \\n'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "09D-rajWKZ7H"
      },
      "source": [
        "start = '2019-10-01' + ' 00:00:00' \n",
        "end = '2019-10-31' + ' 23:00:00' #input('結束日期：(格式:2019-10-17)')\n",
        "time_step = int(96) #int(input('預報小時：(格式:96)'))\n",
        "ver = [17, 18, 30]           #[int(num) for num in input(\"請輸入全部的版本(格式：17 18),以空白鍵區隔：\").split(\",\")] \n",
        "station_num = 74\n",
        "time_interval = time_step//24"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jdigckMZLQXy"
      },
      "source": [
        "def data_cleaning(df):\n",
        "    \n",
        "    df['year'] = df['year'].astype(str)\n",
        "    df['month'] = df['month'].astype(str)\n",
        "    df['day'] = df['day'].astype(str)\n",
        "    df['hour'] = df['hour'].astype(str)\n",
        "    df['dateTime'] = df['year'] + '-' + df['month'].str.zfill(2) + '-' + df['day'].str.zfill(2) + ' ' + df['hour'].str.zfill(2) +':'+ '00:00 '\n",
        "    df['dateTime'] = pd.to_datetime(df['dateTime']).apply(lambda x:int(time.mktime(x.timetuple())))\n",
        "    df.sort_values(by=['dateTime', 'stationID'], inplace = True)\n",
        "\n",
        "    \n",
        "    return df"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gzqYWVYQpbf6"
      },
      "source": [
        "start = time.strptime(start, \"%Y-%m-%d %H:%M:%S\")\n",
        "end = time.strptime(end, \"%Y-%m-%d %H:%M:%S\")\n",
        "\n",
        "# 轉換為時間戳:\n",
        "start = int(time.mktime(start)) - 3600*24\n",
        "end = int(time.mktime(end)) "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gs4OV-Dppbfy"
      },
      "source": [
        "new_observation = pd.read_table(\"opendata_epa.txt\")\n",
        "new_observation = data_cleaning(new_observation)\n",
        "\n",
        "try:\n",
        "    old_observation = pd.read_table(\"old_opendata_epa.txt\")\n",
        "    old_observation = data_cleaning(old_observation)\n",
        "    old_end = old_observation['dateTime'].iloc[-1] + 3600\n",
        "    new_observation = new_observation[new_observation['dateTime']>=old_end]\n",
        "    df_observation = pd.concat([old_observation, new_observation], axis = 0, ignore_index=True)\n",
        "    \n",
        "except IOError:\n",
        "    df_observation = new_observation"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hCcmpr-ypbf8",
        "outputId": "4ccf55ee-a6eb-4627-8bac-09aec92c006a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        }
      },
      "source": [
        "df_observation = df_observation[df_observation['dateTime'].between(start - 3600 * 24, end + (time_interval-1) * 3600 * 24)]\n",
        "df_observation['dateTime'] = df_observation['dateTime'].apply(lambda x:time.strftime(\"%Y-%m-%d %H:%M:%S\",time.localtime(x)))\n",
        "df_observation.sort_values(by=['dateTime', 'stationID'], inplace = True)\n",
        "opendata_station = list(pd.unique(df_observation['stationID']))\n",
        "\n",
        "df_observation.tail()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "C:\\Users\\user\\Anaconda3\\envs\\tf2gpu\\lib\\site-packages\\ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  \n",
            "C:\\Users\\user\\Anaconda3\\envs\\tf2gpu\\lib\\site-packages\\ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  This is separate from the ipykernel package so we can avoid doing imports until\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>year</th>\n",
              "      <th>month</th>\n",
              "      <th>day</th>\n",
              "      <th>hour</th>\n",
              "      <th>stationID</th>\n",
              "      <th>PMf</th>\n",
              "      <th>dateTime</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>553571</th>\n",
              "      <td>2019</td>\n",
              "      <td>10</td>\n",
              "      <td>31</td>\n",
              "      <td>23</td>\n",
              "      <td>EPA084</td>\n",
              "      <td>NaN</td>\n",
              "      <td>2019-10-31 23:00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>553572</th>\n",
              "      <td>2019</td>\n",
              "      <td>10</td>\n",
              "      <td>31</td>\n",
              "      <td>23</td>\n",
              "      <td>EPA101</td>\n",
              "      <td>22.0</td>\n",
              "      <td>2019-10-31 23:00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>553573</th>\n",
              "      <td>2019</td>\n",
              "      <td>10</td>\n",
              "      <td>31</td>\n",
              "      <td>23</td>\n",
              "      <td>EPA102</td>\n",
              "      <td>19.0</td>\n",
              "      <td>2019-10-31 23:00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>553574</th>\n",
              "      <td>2019</td>\n",
              "      <td>10</td>\n",
              "      <td>31</td>\n",
              "      <td>23</td>\n",
              "      <td>EPA104</td>\n",
              "      <td>22.0</td>\n",
              "      <td>2019-10-31 23:00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>553575</th>\n",
              "      <td>2019</td>\n",
              "      <td>10</td>\n",
              "      <td>31</td>\n",
              "      <td>23</td>\n",
              "      <td>EPA110</td>\n",
              "      <td>27.0</td>\n",
              "      <td>2019-10-31 23:00:00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        year month day hour stationID   PMf             dateTime\n",
              "553571  2019    10  31   23    EPA084   NaN  2019-10-31 23:00:00\n",
              "553572  2019    10  31   23    EPA101  22.0  2019-10-31 23:00:00\n",
              "553573  2019    10  31   23    EPA102  19.0  2019-10-31 23:00:00\n",
              "553574  2019    10  31   23    EPA104  22.0  2019-10-31 23:00:00\n",
              "553575  2019    10  31   23    EPA110  27.0  2019-10-31 23:00:00"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ua1LJ85fpbf_"
      },
      "source": [
        "## 移除測站'EPA101',' EPA084',' EPA104'(因為全部皆為NAN)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aO81QQRjpbgA",
        "outputId": "5ac87b33-edf8-412f-f1aa-1006025a9b3e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        }
      },
      "source": [
        "#pd.value_counts(df['stationID'])\n",
        "df_observation = df_observation[df_observation['stationID']!='EPA101']\n",
        "df_observation = df_observation[df_observation['stationID']!='EPA084']\n",
        "df_observation = df_observation[df_observation['stationID']!='EPA104']\n",
        "\n",
        "df_observation.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>year</th>\n",
              "      <th>month</th>\n",
              "      <th>day</th>\n",
              "      <th>hour</th>\n",
              "      <th>stationID</th>\n",
              "      <th>PMf</th>\n",
              "      <th>dateTime</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>492592</th>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>29</td>\n",
              "      <td>0</td>\n",
              "      <td>EPA001</td>\n",
              "      <td>4.0</td>\n",
              "      <td>2019-09-29 00:00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>492593</th>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>29</td>\n",
              "      <td>0</td>\n",
              "      <td>EPA002</td>\n",
              "      <td>4.0</td>\n",
              "      <td>2019-09-29 00:00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>492594</th>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>29</td>\n",
              "      <td>0</td>\n",
              "      <td>EPA003</td>\n",
              "      <td>4.0</td>\n",
              "      <td>2019-09-29 00:00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>492595</th>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>29</td>\n",
              "      <td>0</td>\n",
              "      <td>EPA004</td>\n",
              "      <td>1.0</td>\n",
              "      <td>2019-09-29 00:00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>492596</th>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>29</td>\n",
              "      <td>0</td>\n",
              "      <td>EPA005</td>\n",
              "      <td>5.0</td>\n",
              "      <td>2019-09-29 00:00:00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        year month day hour stationID  PMf             dateTime\n",
              "492592  2019     9  29    0    EPA001  4.0  2019-09-29 00:00:00\n",
              "492593  2019     9  29    0    EPA002  4.0  2019-09-29 00:00:00\n",
              "492594  2019     9  29    0    EPA003  4.0  2019-09-29 00:00:00\n",
              "492595  2019     9  29    0    EPA004  1.0  2019-09-29 00:00:00\n",
              "492596  2019     9  29    0    EPA005  5.0  2019-09-29 00:00:00"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nC_PFCyHbpYD"
      },
      "source": [
        "station_num = len(np.unique(df_observation['stationID'])) ##測站總數"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NU912CU1pbgB"
      },
      "source": [
        "## 進行時間缺漏值填補"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-UiSYrZwpbgD"
      },
      "source": [
        "stations = list(pd.unique(df_observation['stationID']))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5dEZLEaapbgE",
        "outputId": "5b8bdce4-8b69-4882-b772-754d53e41baa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print('------observation 補缺失日期開始------')\n",
        "df_observation['dateTime'] = pd.to_datetime(df_observation['dateTime']).apply(lambda x:int(time.mktime(x.timetuple())))\n",
        "df_observation.index = np.arange(len(df_observation))\n",
        "a = df_observation['dateTime'][0]\n",
        "b = int(df_observation['dateTime'][-1:].values)\n",
        "all_date = list(range(a,b,3600))+[b]\n",
        "df_add = df_observation[: 1].copy()\n",
        "for name in stations:\n",
        "    #print('name', name)\n",
        "    unnormal = list(df_observation[df_observation['stationID'] == name]['dateTime'])\n",
        "    dates = list(set(all_date).difference(set(unnormal)))\n",
        "    #print('date',len(dates))\n",
        "    \n",
        "    for date in dates:\n",
        "        df_add['dateTime'] = date\n",
        "        df_add['stationID'] = name\n",
        "        df_add['PMf'] = np.nan\n",
        "        df_observation = pd.concat([df_observation, df_add])\n",
        "    df_observation.sort_values(by=['dateTime','stationID'], inplace = True)\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "------observation 補缺失日期開始------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c0D3OrcnpbgG",
        "outputId": "f1c550e8-c9ca-4c97-93de-c4d9d2b63604",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        }
      },
      "source": [
        " df_observation['dateTime'] = df_observation['dateTime'].apply(lambda x:time.strftime(\"%Y-%m-%d %H:%M:%S\",time.localtime(x)))\n",
        "df_observation['month'] = pd.to_datetime(df_observation['dateTime']).dt.month\n",
        "df_observation['year'] = pd.to_datetime(df_observation['dateTime']).dt.year\n",
        "df_observation['day'] = pd.to_datetime(df_observation['dateTime']).dt.day\n",
        "df_observation['hour'] = pd.to_datetime(df_observation['dateTime']).dt.hour\n",
        "\n",
        "df_observation.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>year</th>\n",
              "      <th>month</th>\n",
              "      <th>day</th>\n",
              "      <th>hour</th>\n",
              "      <th>stationID</th>\n",
              "      <th>PMf</th>\n",
              "      <th>dateTime</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>29</td>\n",
              "      <td>0</td>\n",
              "      <td>EPA001</td>\n",
              "      <td>4.0</td>\n",
              "      <td>2019-09-29 00:00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>29</td>\n",
              "      <td>0</td>\n",
              "      <td>EPA002</td>\n",
              "      <td>4.0</td>\n",
              "      <td>2019-09-29 00:00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>29</td>\n",
              "      <td>0</td>\n",
              "      <td>EPA003</td>\n",
              "      <td>4.0</td>\n",
              "      <td>2019-09-29 00:00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>29</td>\n",
              "      <td>0</td>\n",
              "      <td>EPA004</td>\n",
              "      <td>1.0</td>\n",
              "      <td>2019-09-29 00:00:00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>29</td>\n",
              "      <td>0</td>\n",
              "      <td>EPA005</td>\n",
              "      <td>5.0</td>\n",
              "      <td>2019-09-29 00:00:00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   year  month  day  hour stationID  PMf             dateTime\n",
              "0  2019      9   29     0    EPA001  4.0  2019-09-29 00:00:00\n",
              "1  2019      9   29     0    EPA002  4.0  2019-09-29 00:00:00\n",
              "2  2019      9   29     0    EPA003  4.0  2019-09-29 00:00:00\n",
              "3  2019      9   29     0    EPA004  1.0  2019-09-29 00:00:00\n",
              "4  2019      9   29     0    EPA005  5.0  2019-09-29 00:00:00"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "161URL2BpbgH"
      },
      "source": [
        "### 檢查補植後結果 1:補值成功 ; 不為1:補值失敗"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qSoK8PjXpbgI",
        "outputId": "e3ee44d6-f615-4e33-94f1-3a90eb8fffc7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "if len(np.unique(pd.value_counts(df_observation['stationID'])))==1:\n",
        "    print(\"observation補缺失日期成功\")\n",
        "else:\n",
        "    print(\"observation補缺失日期失敗\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "observation補缺失日期成功\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nnyhlW57IWdA"
      },
      "source": [
        "df_observation.to_csv('old_corr_opendata_epa.txt', sep='\\t', index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gUMBGkmMPJ2e"
      },
      "source": [
        "df_observation['date'] = df_observation['dateTime'].apply(lambda x:time.strftime(\"%Y-%m-%d\",time.strptime(x,'%Y-%m-%d %H:%M:%S')))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wyCYtTTbpbgK"
      },
      "source": [
        "## 轉為一天為單位的矩陣"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "keKt7ZRhpbgO",
        "outputId": "f6337bcb-3975-4a1c-e5ca-a49407d4cce8"
      },
      "source": [
        "one_day_df = np.array(df_observation['PMf']).reshape(-1, station_num * 24)\n",
        "one_day_df"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[ 4.,  4.,  4., ...,  3., 30., 11.],\n",
              "       [ 8.,  8.,  7., ...,  7.,  9., 13.],\n",
              "       [ 6.,  5.,  8., ...,  3., 39., 23.],\n",
              "       ...,\n",
              "       [ 6.,  6.,  3., ...,  0., 61., 18.],\n",
              "       [19., 19., 24., ...,  7., 31., 24.],\n",
              "       [27., 26., 40., ...,  4., 19., 27.]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bsiR4aXxzgVX",
        "outputId": "5b2876f8-49e9-4d07-c03f-8f908c2453ed"
      },
      "source": [
        "index = pd.unique(df_observation['date'] ).astype(str)\n",
        "observation_z = pd.DataFrame(columns=np.arange(station_num*24), data = one_day_df,index = index)\n",
        "observation_y1 =  observation_z.copy()\n",
        "observation_z.tail()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>...</th>\n",
              "      <th>1766</th>\n",
              "      <th>1767</th>\n",
              "      <th>1768</th>\n",
              "      <th>1769</th>\n",
              "      <th>1770</th>\n",
              "      <th>1771</th>\n",
              "      <th>1772</th>\n",
              "      <th>1773</th>\n",
              "      <th>1774</th>\n",
              "      <th>1775</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>2019-10-27</td>\n",
              "      <td>15.0</td>\n",
              "      <td>20.0</td>\n",
              "      <td>14.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>19.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>20.0</td>\n",
              "      <td>24.0</td>\n",
              "      <td>20.0</td>\n",
              "      <td>...</td>\n",
              "      <td>4.0</td>\n",
              "      <td>26.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>23.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>27.0</td>\n",
              "      <td>33.0</td>\n",
              "      <td>13.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>13.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-28</td>\n",
              "      <td>8.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>11.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>14.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>...</td>\n",
              "      <td>4.0</td>\n",
              "      <td>16.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>13.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>14.0</td>\n",
              "      <td>21.0</td>\n",
              "      <td>11.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>7.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-29</td>\n",
              "      <td>6.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>...</td>\n",
              "      <td>17.0</td>\n",
              "      <td>25.0</td>\n",
              "      <td>17.0</td>\n",
              "      <td>21.0</td>\n",
              "      <td>19.0</td>\n",
              "      <td>25.0</td>\n",
              "      <td>25.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>61.0</td>\n",
              "      <td>18.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-30</td>\n",
              "      <td>19.0</td>\n",
              "      <td>19.0</td>\n",
              "      <td>24.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>18.0</td>\n",
              "      <td>7.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>21.0</td>\n",
              "      <td>18.0</td>\n",
              "      <td>...</td>\n",
              "      <td>12.0</td>\n",
              "      <td>42.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>45.0</td>\n",
              "      <td>25.0</td>\n",
              "      <td>61.0</td>\n",
              "      <td>60.0</td>\n",
              "      <td>7.0</td>\n",
              "      <td>31.0</td>\n",
              "      <td>24.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-31</td>\n",
              "      <td>27.0</td>\n",
              "      <td>26.0</td>\n",
              "      <td>40.0</td>\n",
              "      <td>24.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>32.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>32.0</td>\n",
              "      <td>34.0</td>\n",
              "      <td>27.0</td>\n",
              "      <td>...</td>\n",
              "      <td>25.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>21.0</td>\n",
              "      <td>32.0</td>\n",
              "      <td>20.0</td>\n",
              "      <td>37.0</td>\n",
              "      <td>43.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>19.0</td>\n",
              "      <td>27.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 1776 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "            0     1     2     3     4     5     6     7     8     9     ...  \\\n",
              "2019-10-27  15.0  20.0  14.0  15.0  15.0  19.0  15.0  20.0  24.0  20.0  ...   \n",
              "2019-10-28   8.0   9.0   8.0   5.0  11.0  10.0  10.0  14.0  15.0  12.0  ...   \n",
              "2019-10-29   6.0   6.0   3.0   3.0   4.0   2.0   5.0   8.0   5.0   6.0  ...   \n",
              "2019-10-30  19.0  19.0  24.0  15.0  18.0   7.0  12.0  12.0  21.0  18.0  ...   \n",
              "2019-10-31  27.0  26.0  40.0  24.0  29.0  32.0  29.0  32.0  34.0  27.0  ...   \n",
              "\n",
              "            1766  1767  1768  1769  1770  1771  1772  1773  1774  1775  \n",
              "2019-10-27   4.0  26.0   9.0  23.0   8.0  27.0  33.0  13.0  10.0  13.0  \n",
              "2019-10-28   4.0  16.0   9.0  13.0   3.0  14.0  21.0  11.0   2.0   7.0  \n",
              "2019-10-29  17.0  25.0  17.0  21.0  19.0  25.0  25.0   0.0  61.0  18.0  \n",
              "2019-10-30  12.0  42.0  29.0  45.0  25.0  61.0  60.0   7.0  31.0  24.0  \n",
              "2019-10-31  25.0  29.0  21.0  32.0  20.0  37.0  43.0   4.0  19.0  27.0  \n",
              "\n",
              "[5 rows x 1776 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kaj3rvESbpYR"
      },
      "source": [
        "## 將PMF=NAN進行補值"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EyHfX7Q3bpYS"
      },
      "source": [
        "### 引入R的套件"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PAC7ek6qbpYS"
      },
      "source": [
        "#%load_ext rpy2.ipython"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RcCV-1x-bpYU",
        "outputId": "97ad447e-3a92-4a02-e43e-688c6ed45bbd"
      },
      "source": [
        "robjects.r(\"install.packages('devtools')\")\n",
        "\n",
        "robjects.r('source(\"https://bioconductor.org/biocLite.R\")')\n",
        "r('''devtools::install_github(\"marchtaylor/sinkr\")''')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "    <span>BoolVector with 1 elements.</span>\n",
              "    <table>\n",
              "      <tbody>\n",
              "      <tr>\n",
              "      \n",
              "      <td>\n",
              "               0\n",
              "      </td>\n",
              "      \n",
              "      </tr>\n",
              "      </tbody>\n",
              "    </table>\n",
              "    "
            ],
            "text/plain": [
              "R object with classes: ('logical',) mapped to:\n",
              "<BoolVector - Python:0x00000217907D38C8 / R:0x000002178D5C9618>\n",
              "[       0]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZBC1DS2PbpYX"
      },
      "source": [
        "devtools = importr('devtools')\n",
        "sinkr = importr('sinkr')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1UBn8hA2bpYZ",
        "outputId": "0de39ea4-2d4f-4e2c-b3d8-46f848d2cfd0"
      },
      "source": [
        "numpy2ri.activate()\n",
        "pandas2ri.activate()\n",
        "rpy2_observation_z = pandas2ri.py2ri(observation_z)\n",
        "robjects.r.assign('rpy2_observation_z', rpy2_observation_z)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "    <span>R/rpy2 DataFrame (33 x 1776)</span>\n",
              "    <table>\n",
              "      <thead>\n",
              "        <tr>\n",
              "        \n",
              "          <th>X0</th>\n",
              "        \n",
              "          <th>X1</th>\n",
              "        \n",
              "          <th>X2</th>\n",
              "        \n",
              "          <th>...</th>\n",
              "        \n",
              "          <th>X1773</th>\n",
              "        \n",
              "          <th>X1774</th>\n",
              "        \n",
              "          <th>X1775</th>\n",
              "        \n",
              "        </tr>\n",
              "      </thead>\n",
              "      <tbody>\n",
              "      \n",
              "      <tr>\n",
              "      \n",
              "      <td>\n",
              "        4.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        4.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        4.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        3.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        30.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        11.000000\n",
              "      </td>\n",
              "      \n",
              "      </tr>\n",
              "      \n",
              "      <tr>\n",
              "      \n",
              "      <td>\n",
              "        8.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        8.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        7.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        7.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        9.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        13.000000\n",
              "      </td>\n",
              "      \n",
              "      </tr>\n",
              "      \n",
              "      <tr>\n",
              "      \n",
              "      <td>\n",
              "        6.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        5.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        8.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        3.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        39.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        23.000000\n",
              "      </td>\n",
              "      \n",
              "      </tr>\n",
              "      \n",
              "      <tr>\n",
              "      \n",
              "      <td>\n",
              "        10.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        17.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        9.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        12.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        62.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        52.000000\n",
              "      </td>\n",
              "      \n",
              "      </tr>\n",
              "      \n",
              "      <tr>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      </tr>\n",
              "      \n",
              "      <tr>\n",
              "      \n",
              "      <td>\n",
              "        8.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        9.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        8.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        11.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        2.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        7.000000\n",
              "      </td>\n",
              "      \n",
              "      </tr>\n",
              "      \n",
              "      <tr>\n",
              "      \n",
              "      <td>\n",
              "        6.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        6.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        3.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        0.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        61.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        18.000000\n",
              "      </td>\n",
              "      \n",
              "      </tr>\n",
              "      \n",
              "      <tr>\n",
              "      \n",
              "      <td>\n",
              "        19.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        19.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        24.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        7.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        31.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        24.000000\n",
              "      </td>\n",
              "      \n",
              "      </tr>\n",
              "      \n",
              "      <tr>\n",
              "      \n",
              "      <td>\n",
              "        27.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        26.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        40.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        4.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        19.000000\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        27.000000\n",
              "      </td>\n",
              "      \n",
              "      </tr>\n",
              "      \n",
              "      </tbody>\n",
              "    </table>\n",
              "    "
            ],
            "text/plain": [
              "R object with classes: ('data.frame',) mapped to:\n",
              "<DataFrame - Python:0x0000021791EFF908 / R:0x00000217951074F8>\n",
              "[FloatVector, FloatVector, FloatVector, FloatVector, ..., FloatVector, FloatVector, FloatVector, FloatVector]\n",
              "  X0: <class 'rpy2.robjects.vectors.FloatVector'>\n",
              "  R object with classes: ('numeric',) mapped to:\n",
              "<FloatVector - Python:0x0000021794292848 / R:0x000002178D888C68>\n",
              "[4.000000, 8.000000, 6.000000, 10.000000, ..., 8.000000, 6.000000, 19.000000, 27.000000]\n",
              "  X1: <class 'rpy2.robjects.vectors.FloatVector'>\n",
              "  R object with classes: ('numeric',) mapped to:\n",
              "<FloatVector - Python:0x0000021794256C48 / R:0x000002178E81AB20>\n",
              "[4.000000, 8.000000, 5.000000, 17.000000, ..., 9.000000, 6.000000, 19.000000, 26.000000]\n",
              "  X2: <class 'rpy2.robjects.vectors.FloatVector'>\n",
              "  R object with classes: ('numeric',) mapped to:\n",
              "<FloatVector - Python:0x00000217941FA288 / R:0x000002178E755E98>\n",
              "[4.000000, 7.000000, 8.000000, 9.000000, ..., 8.000000, 3.000000, 24.000000, 40.000000]\n",
              "  X3: <class 'rpy2.robjects.vectors.FloatVector'>\n",
              "  R object with classes: ('numeric',) mapped to:\n",
              "<FloatVector - Python:0x00000217941FA388 / R:0x000002178D6A38F8>\n",
              "[1.000000, 5.000000, 1.000000, 5.000000, ..., 5.000000, 3.000000, 15.000000, 24.000000]\n",
              "...\n",
              "  X5: <class 'rpy2.robjects.vectors.FloatVector'>\n",
              "  R object with classes: ('numeric',) mapped to:\n",
              "<FloatVector - Python:0x00000217941FA508 / R:0x000002178E1B4D08>\n",
              "[29.000000, 5.000000, 9.000000, 41.000000, ..., 21.000000, 25.000000, 60.000000, 43.000000]\n",
              "  X6: <class 'rpy2.robjects.vectors.FloatVector'>\n",
              "  R object with classes: ('numeric',) mapped to:\n",
              "<FloatVector - Python:0x00000217941FA608 / R:0x000002178D89A718>\n",
              "[3.000000, 7.000000, 3.000000, 12.000000, ..., 11.000000, 0.000000, 7.000000, 4.000000]\n",
              "  X7: <class 'rpy2.robjects.vectors.FloatVector'>\n",
              "  R object with classes: ('numeric',) mapped to:\n",
              "<FloatVector - Python:0x0000021794292848 / R:0x000002178D89AB50>\n",
              "[30.000000, 9.000000, 39.000000, 62.000000, ..., 2.000000, 61.000000, 31.000000, 19.000000]\n",
              "  X8: <class 'rpy2.robjects.vectors.FloatVector'>\n",
              "  R object with classes: ('numeric',) mapped to:\n",
              "<FloatVector - Python:0x00000217941FA588 / R:0x000002178D89AF88>\n",
              "[11.000000, 13.000000, 23.000000, 52.000000, ..., 7.000000, 18.000000, 24.000000, 27.000000]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bJFVN1KabpYb",
        "outputId": "93aad563-35c4-4186-ac34-025b43054b37"
      },
      "source": [
        "r('''\n",
        "rpy2_observation_z[is.na(rpy2_observation_z)]<- NaN\n",
        "y <- data.matrix(rpy2_observation_z)\n",
        "Ya <- dineof(y)$Xa\n",
        "observation_z<-data.frame(Ya)\n",
        "head(observation_z)\n",
        "''')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "C:\\Users\\User\\Anaconda3\\envs\\tf2_rpy2\\lib\\site-packages\\rpy2-2.9.4-py3.6-win-amd64.egg\\rpy2\\robjects\\pandas2ri.py:191: FutureWarning: from_items is deprecated. Please use DataFrame.from_dict(dict(items), ...) instead. DataFrame.from_dict(OrderedDict(items)) may be used to preserve the key order.\n",
            "  res = PandasDataFrame.from_items(items)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>X0</th>\n",
              "      <th>X1</th>\n",
              "      <th>X2</th>\n",
              "      <th>X3</th>\n",
              "      <th>X4</th>\n",
              "      <th>X5</th>\n",
              "      <th>X6</th>\n",
              "      <th>X7</th>\n",
              "      <th>X8</th>\n",
              "      <th>X9</th>\n",
              "      <th>...</th>\n",
              "      <th>X1766</th>\n",
              "      <th>X1767</th>\n",
              "      <th>X1768</th>\n",
              "      <th>X1769</th>\n",
              "      <th>X1770</th>\n",
              "      <th>X1771</th>\n",
              "      <th>X1772</th>\n",
              "      <th>X1773</th>\n",
              "      <th>X1774</th>\n",
              "      <th>X1775</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>...</td>\n",
              "      <td>8.000000</td>\n",
              "      <td>18.0</td>\n",
              "      <td>11.0</td>\n",
              "      <td>23.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>18.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>30.0</td>\n",
              "      <td>11.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>8.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>7.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>11.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>7.0</td>\n",
              "      <td>...</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>18.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>7.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>13.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>6.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>7.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>7.0</td>\n",
              "      <td>...</td>\n",
              "      <td>3.000000</td>\n",
              "      <td>26.0</td>\n",
              "      <td>13.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>7.0</td>\n",
              "      <td>33.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>39.0</td>\n",
              "      <td>23.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>10.0</td>\n",
              "      <td>17.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>13.0</td>\n",
              "      <td>16.0</td>\n",
              "      <td>16.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>...</td>\n",
              "      <td>8.000000</td>\n",
              "      <td>43.0</td>\n",
              "      <td>59.0</td>\n",
              "      <td>37.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>48.0</td>\n",
              "      <td>41.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>62.0</td>\n",
              "      <td>52.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>25.0</td>\n",
              "      <td>23.0</td>\n",
              "      <td>18.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>36.0</td>\n",
              "      <td>30.0</td>\n",
              "      <td>26.0</td>\n",
              "      <td>16.0</td>\n",
              "      <td>43.0</td>\n",
              "      <td>43.0</td>\n",
              "      <td>...</td>\n",
              "      <td>23.000000</td>\n",
              "      <td>53.0</td>\n",
              "      <td>31.0</td>\n",
              "      <td>28.0</td>\n",
              "      <td>36.0</td>\n",
              "      <td>21.0</td>\n",
              "      <td>32.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>38.0</td>\n",
              "      <td>42.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>27.0</td>\n",
              "      <td>41.0</td>\n",
              "      <td>23.0</td>\n",
              "      <td>21.0</td>\n",
              "      <td>31.0</td>\n",
              "      <td>34.0</td>\n",
              "      <td>37.0</td>\n",
              "      <td>34.0</td>\n",
              "      <td>45.0</td>\n",
              "      <td>38.0</td>\n",
              "      <td>...</td>\n",
              "      <td>7.676123</td>\n",
              "      <td>26.0</td>\n",
              "      <td>36.0</td>\n",
              "      <td>40.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>14.0</td>\n",
              "      <td>59.0</td>\n",
              "      <td>17.0</td>\n",
              "      <td>22.0</td>\n",
              "      <td>38.930303</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>6 rows × 1776 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "     X0    X1    X2    X3    X4    X5    X6    X7    X8    X9  ...      X1766  \\\n",
              "0   4.0   4.0   4.0   1.0   5.0   1.0   6.0   6.0   8.0   5.0  ...   8.000000   \n",
              "1   8.0   8.0   7.0   5.0  10.0   4.0  11.0   9.0  10.0   7.0  ...   1.000000   \n",
              "2   6.0   5.0   8.0   1.0   5.0  10.0   7.0   4.0   6.0   7.0  ...   3.000000   \n",
              "3  10.0  17.0   9.0   5.0  13.0  16.0  16.0   8.0   9.0  15.0  ...   8.000000   \n",
              "4  25.0  23.0  18.0   8.0  36.0  30.0  26.0  16.0  43.0  43.0  ...  23.000000   \n",
              "5  27.0  41.0  23.0  21.0  31.0  34.0  37.0  34.0  45.0  38.0  ...   7.676123   \n",
              "\n",
              "   X1767  X1768  X1769  X1770  X1771  X1772  X1773  X1774      X1775  \n",
              "0   18.0   11.0   23.0    6.0   18.0   29.0    3.0   30.0  11.000000  \n",
              "1   18.0    9.0    3.0    4.0   12.0    5.0    7.0    9.0  13.000000  \n",
              "2   26.0   13.0   12.0    7.0   33.0    9.0    3.0   39.0  23.000000  \n",
              "3   43.0   59.0   37.0    9.0   48.0   41.0   12.0   62.0  52.000000  \n",
              "4   53.0   31.0   28.0   36.0   21.0   32.0    8.0   38.0  42.000000  \n",
              "5   26.0   36.0   40.0    9.0   14.0   59.0   17.0   22.0  38.930303  \n",
              "\n",
              "[6 rows x 1776 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7Ld5tg99pbgU"
      },
      "source": [
        "## 補值過後的observation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qKPj5H_ebpYd"
      },
      "source": [
        "#### R dataframe 轉換 pandas"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-fh-KBY1bpYd",
        "outputId": "9eb3901e-1802-4c2f-9a50-a5bbe763a705"
      },
      "source": [
        "r.data('observation_z')\n",
        "df_nomissing = pd.DataFrame(r['observation_z'])\n",
        "df_nomissing.tail()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>X0</th>\n",
              "      <th>X1</th>\n",
              "      <th>X2</th>\n",
              "      <th>X3</th>\n",
              "      <th>X4</th>\n",
              "      <th>X5</th>\n",
              "      <th>X6</th>\n",
              "      <th>X7</th>\n",
              "      <th>X8</th>\n",
              "      <th>X9</th>\n",
              "      <th>...</th>\n",
              "      <th>X1766</th>\n",
              "      <th>X1767</th>\n",
              "      <th>X1768</th>\n",
              "      <th>X1769</th>\n",
              "      <th>X1770</th>\n",
              "      <th>X1771</th>\n",
              "      <th>X1772</th>\n",
              "      <th>X1773</th>\n",
              "      <th>X1774</th>\n",
              "      <th>X1775</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>28</td>\n",
              "      <td>15.0</td>\n",
              "      <td>20.0</td>\n",
              "      <td>14.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>19.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>20.0</td>\n",
              "      <td>24.0</td>\n",
              "      <td>20.0</td>\n",
              "      <td>...</td>\n",
              "      <td>4.0</td>\n",
              "      <td>26.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>23.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>27.0</td>\n",
              "      <td>33.0</td>\n",
              "      <td>13.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>13.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>29</td>\n",
              "      <td>8.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>11.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>14.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>...</td>\n",
              "      <td>4.0</td>\n",
              "      <td>16.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>13.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>14.0</td>\n",
              "      <td>21.0</td>\n",
              "      <td>11.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>7.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>30</td>\n",
              "      <td>6.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>...</td>\n",
              "      <td>17.0</td>\n",
              "      <td>25.0</td>\n",
              "      <td>17.0</td>\n",
              "      <td>21.0</td>\n",
              "      <td>19.0</td>\n",
              "      <td>25.0</td>\n",
              "      <td>25.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>61.0</td>\n",
              "      <td>18.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>31</td>\n",
              "      <td>19.0</td>\n",
              "      <td>19.0</td>\n",
              "      <td>24.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>18.0</td>\n",
              "      <td>7.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>21.0</td>\n",
              "      <td>18.0</td>\n",
              "      <td>...</td>\n",
              "      <td>12.0</td>\n",
              "      <td>42.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>45.0</td>\n",
              "      <td>25.0</td>\n",
              "      <td>61.0</td>\n",
              "      <td>60.0</td>\n",
              "      <td>7.0</td>\n",
              "      <td>31.0</td>\n",
              "      <td>24.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>32</td>\n",
              "      <td>27.0</td>\n",
              "      <td>26.0</td>\n",
              "      <td>40.0</td>\n",
              "      <td>24.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>32.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>32.0</td>\n",
              "      <td>34.0</td>\n",
              "      <td>27.0</td>\n",
              "      <td>...</td>\n",
              "      <td>25.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>21.0</td>\n",
              "      <td>32.0</td>\n",
              "      <td>20.0</td>\n",
              "      <td>37.0</td>\n",
              "      <td>43.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>19.0</td>\n",
              "      <td>27.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 1776 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "      X0    X1    X2    X3    X4    X5    X6    X7    X8    X9  ...  X1766  \\\n",
              "28  15.0  20.0  14.0  15.0  15.0  19.0  15.0  20.0  24.0  20.0  ...    4.0   \n",
              "29   8.0   9.0   8.0   5.0  11.0  10.0  10.0  14.0  15.0  12.0  ...    4.0   \n",
              "30   6.0   6.0   3.0   3.0   4.0   2.0   5.0   8.0   5.0   6.0  ...   17.0   \n",
              "31  19.0  19.0  24.0  15.0  18.0   7.0  12.0  12.0  21.0  18.0  ...   12.0   \n",
              "32  27.0  26.0  40.0  24.0  29.0  32.0  29.0  32.0  34.0  27.0  ...   25.0   \n",
              "\n",
              "    X1767  X1768  X1769  X1770  X1771  X1772  X1773  X1774  X1775  \n",
              "28   26.0    9.0   23.0    8.0   27.0   33.0   13.0   10.0   13.0  \n",
              "29   16.0    9.0   13.0    3.0   14.0   21.0   11.0    2.0    7.0  \n",
              "30   25.0   17.0   21.0   19.0   25.0   25.0    0.0   61.0   18.0  \n",
              "31   42.0   29.0   45.0   25.0   61.0   60.0    7.0   31.0   24.0  \n",
              "32   29.0   21.0   32.0   20.0   37.0   43.0    4.0   19.0   27.0  \n",
              "\n",
              "[5 rows x 1776 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DEzsMo_6pbga",
        "outputId": "eff416ee-8cf8-4af4-85f1-2de38e156321"
      },
      "source": [
        "index = pd.unique(df_observation['date'])\n",
        "observation_z = pd.DataFrame(columns=np.arange(station_num*time_step), data = np.nan, index = index)\n",
        "observation_z.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>...</th>\n",
              "      <th>7094</th>\n",
              "      <th>7095</th>\n",
              "      <th>7096</th>\n",
              "      <th>7097</th>\n",
              "      <th>7098</th>\n",
              "      <th>7099</th>\n",
              "      <th>7100</th>\n",
              "      <th>7101</th>\n",
              "      <th>7102</th>\n",
              "      <th>7103</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>2019-09-29</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-09-30</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-01</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-02</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-03</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 7104 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "            0     1     2     3     4     5     6     7     8     9     ...  \\\n",
              "2019-09-29   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
              "2019-09-30   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
              "2019-10-01   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
              "2019-10-02   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
              "2019-10-03   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
              "\n",
              "            7094  7095  7096  7097  7098  7099  7100  7101  7102  7103  \n",
              "2019-09-29   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
              "2019-09-30   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
              "2019-10-01   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
              "2019-10-02   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
              "2019-10-03   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
              "\n",
              "[5 rows x 7104 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1Jdffgewpbgd",
        "outputId": "184ddb82-4bde-4830-84b0-5067a838444a"
      },
      "source": [
        "j = 1\n",
        "df_nomissing = np.array(df_nomissing)\n",
        "for i in range(len(observation_z)):\n",
        "    try:\n",
        "        observation_z.iloc[i,:]  = np.vstack(df_nomissing[i + np.arange(time_interval)]).flatten()\n",
        "    except:\n",
        "        observation_z.iloc[i,:]  = np.vstack([df_nomissing[i + np.arange(time_interval-j)],np.zeros_like(df_nomissing[:j]) * np.nan]).flatten()\n",
        "        j = j + 1\n",
        "observation_z.tail()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>...</th>\n",
              "      <th>7094</th>\n",
              "      <th>7095</th>\n",
              "      <th>7096</th>\n",
              "      <th>7097</th>\n",
              "      <th>7098</th>\n",
              "      <th>7099</th>\n",
              "      <th>7100</th>\n",
              "      <th>7101</th>\n",
              "      <th>7102</th>\n",
              "      <th>7103</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>2019-10-27</td>\n",
              "      <td>15.0</td>\n",
              "      <td>20.0</td>\n",
              "      <td>14.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>19.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>20.0</td>\n",
              "      <td>24.0</td>\n",
              "      <td>20.0</td>\n",
              "      <td>...</td>\n",
              "      <td>12.0</td>\n",
              "      <td>42.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>45.0</td>\n",
              "      <td>25.0</td>\n",
              "      <td>61.0</td>\n",
              "      <td>60.0</td>\n",
              "      <td>7.0</td>\n",
              "      <td>31.0</td>\n",
              "      <td>24.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-28</td>\n",
              "      <td>8.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>11.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>14.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>...</td>\n",
              "      <td>25.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>21.0</td>\n",
              "      <td>32.0</td>\n",
              "      <td>20.0</td>\n",
              "      <td>37.0</td>\n",
              "      <td>43.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>19.0</td>\n",
              "      <td>27.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-29</td>\n",
              "      <td>6.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-30</td>\n",
              "      <td>19.0</td>\n",
              "      <td>19.0</td>\n",
              "      <td>24.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>18.0</td>\n",
              "      <td>7.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>21.0</td>\n",
              "      <td>18.0</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-31</td>\n",
              "      <td>27.0</td>\n",
              "      <td>26.0</td>\n",
              "      <td>40.0</td>\n",
              "      <td>24.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>32.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>32.0</td>\n",
              "      <td>34.0</td>\n",
              "      <td>27.0</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 7104 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "            0     1     2     3     4     5     6     7     8     9     ...  \\\n",
              "2019-10-27  15.0  20.0  14.0  15.0  15.0  19.0  15.0  20.0  24.0  20.0  ...   \n",
              "2019-10-28   8.0   9.0   8.0   5.0  11.0  10.0  10.0  14.0  15.0  12.0  ...   \n",
              "2019-10-29   6.0   6.0   3.0   3.0   4.0   2.0   5.0   8.0   5.0   6.0  ...   \n",
              "2019-10-30  19.0  19.0  24.0  15.0  18.0   7.0  12.0  12.0  21.0  18.0  ...   \n",
              "2019-10-31  27.0  26.0  40.0  24.0  29.0  32.0  29.0  32.0  34.0  27.0  ...   \n",
              "\n",
              "            7094  7095  7096  7097  7098  7099  7100  7101  7102  7103  \n",
              "2019-10-27  12.0  42.0  29.0  45.0  25.0  61.0  60.0   7.0  31.0  24.0  \n",
              "2019-10-28  25.0  29.0  21.0  32.0  20.0  37.0  43.0   4.0  19.0  27.0  \n",
              "2019-10-29   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
              "2019-10-30   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
              "2019-10-31   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
              "\n",
              "[5 rows x 7104 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KfK7v0QTzgVt"
      },
      "source": [
        "## Observation_y"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dOvM301wzgVt",
        "outputId": "4692e810-9473-4590-964a-bfddee97821b"
      },
      "source": [
        "observation_y = pd.DataFrame(columns=np.arange(station_num * time_step), data = np.nan,index = index)\n",
        "observation_y1 = np.array(observation_y1)\n",
        "j = 1\n",
        "df_nomissing = np.array(df_nomissing)\n",
        "for i in range(len(observation_z)):\n",
        "    try:\n",
        "        observation_y.iloc[i,:]  = np.vstack(observation_y1[i + np.arange(time_interval)]).flatten()\n",
        "    except:\n",
        "        observation_y.iloc[i,:]  = np.vstack([observation_y1[i + np.arange(time_interval-j)],np.zeros_like(observation_y1[:j]) * np.nan]).flatten()\n",
        "        j = j + 1\n",
        "observation_y.tail()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>...</th>\n",
              "      <th>7094</th>\n",
              "      <th>7095</th>\n",
              "      <th>7096</th>\n",
              "      <th>7097</th>\n",
              "      <th>7098</th>\n",
              "      <th>7099</th>\n",
              "      <th>7100</th>\n",
              "      <th>7101</th>\n",
              "      <th>7102</th>\n",
              "      <th>7103</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>2019-10-27</td>\n",
              "      <td>15.0</td>\n",
              "      <td>20.0</td>\n",
              "      <td>14.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>19.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>20.0</td>\n",
              "      <td>24.0</td>\n",
              "      <td>20.0</td>\n",
              "      <td>...</td>\n",
              "      <td>12.0</td>\n",
              "      <td>42.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>45.0</td>\n",
              "      <td>25.0</td>\n",
              "      <td>61.0</td>\n",
              "      <td>60.0</td>\n",
              "      <td>7.0</td>\n",
              "      <td>31.0</td>\n",
              "      <td>24.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-28</td>\n",
              "      <td>8.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>11.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>14.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>...</td>\n",
              "      <td>25.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>21.0</td>\n",
              "      <td>32.0</td>\n",
              "      <td>20.0</td>\n",
              "      <td>37.0</td>\n",
              "      <td>43.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>19.0</td>\n",
              "      <td>27.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-29</td>\n",
              "      <td>6.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-30</td>\n",
              "      <td>19.0</td>\n",
              "      <td>19.0</td>\n",
              "      <td>24.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>18.0</td>\n",
              "      <td>7.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>21.0</td>\n",
              "      <td>18.0</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-31</td>\n",
              "      <td>27.0</td>\n",
              "      <td>26.0</td>\n",
              "      <td>40.0</td>\n",
              "      <td>24.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>32.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>32.0</td>\n",
              "      <td>34.0</td>\n",
              "      <td>27.0</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 7104 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "            0     1     2     3     4     5     6     7     8     9     ...  \\\n",
              "2019-10-27  15.0  20.0  14.0  15.0  15.0  19.0  15.0  20.0  24.0  20.0  ...   \n",
              "2019-10-28   8.0   9.0   8.0   5.0  11.0  10.0  10.0  14.0  15.0  12.0  ...   \n",
              "2019-10-29   6.0   6.0   3.0   3.0   4.0   2.0   5.0   8.0   5.0   6.0  ...   \n",
              "2019-10-30  19.0  19.0  24.0  15.0  18.0   7.0  12.0  12.0  21.0  18.0  ...   \n",
              "2019-10-31  27.0  26.0  40.0  24.0  29.0  32.0  29.0  32.0  34.0  27.0  ...   \n",
              "\n",
              "            7094  7095  7096  7097  7098  7099  7100  7101  7102  7103  \n",
              "2019-10-27  12.0  42.0  29.0  45.0  25.0  61.0  60.0   7.0  31.0  24.0  \n",
              "2019-10-28  25.0  29.0  21.0  32.0  20.0  37.0  43.0   4.0  19.0  27.0  \n",
              "2019-10-29   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
              "2019-10-30   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
              "2019-10-31   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
              "\n",
              "[5 rows x 7104 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CdGlyMZjbpYi"
      },
      "source": [
        "# ----------------------------------------------------------------------------------------"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p8ffuXZVbpYi"
      },
      "source": [
        "# Prediction"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y3KM9kPQbpYk"
      },
      "source": [
        "def getEveryDay(begin_date,end_date):\n",
        "    date_list = []\n",
        "    begin_date = datetime.strptime(begin_date, \"%Y-%m-%d\")\n",
        "    end_date = datetime.strptime(end_date,\"%Y-%m-%d\")\n",
        "    while begin_date <= end_date:\n",
        "        date_str = begin_date.strftime(\"%Y-%m-%d\")\n",
        "        date_list.append(date_str)\n",
        "        begin_date += timedelta(days=1)\n",
        "        \n",
        "    return date_list"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BrWAPKLKLQYm"
      },
      "source": [
        "def data_cleaning_pred(df):\n",
        "    df['expdate'] = pd.to_datetime(df['expdate']).apply(lambda x:int(time.mktime(x.timetuple())))\n",
        "    return df"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MZCEfeq7bpYm"
      },
      "source": [
        "name = 'stat_v' \n",
        "for i in ver:\n",
        "    new_name = name + str(i) + '.txt'\n",
        "    prediction = pd.read_table(new_name)\n",
        "    if i == ver[0]:\n",
        "        new_prediction = prediction\n",
        "    else:\n",
        "        new_prediction = pd.concat([new_prediction, prediction], axis=0, sort = False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kro_nZcXLQYq",
        "outputId": "6693b1c2-666f-45b4-a180-e06f0f14384c"
      },
      "source": [
        "new_prediction = data_cleaning_pred(new_prediction)\n",
        "file_name = 'corr_old_preciction_' + str(time_step) + '.txt'\n",
        "\n",
        "try:\n",
        "    old_prediction = pd.read_table(file_name)\n",
        "    old_prediction = data_cleaning_pred(old_prediction)\n",
        "    old_end = old_prediction['expdate'].iloc[-1] + 3600\n",
        "    new_prediction = new_prediction[new_prediction['expdate']>=old_end]\n",
        "    df_prediction = pd.concat([old_prediction, new_prediction], axis = 0, ignore_index=True)\n",
        "    \n",
        "except IOError:\n",
        "    df_prediction = new_prediction\n",
        "    \n",
        "df_prediction = df_prediction[df_prediction['expdate'].between(start - 3600*24, end - 3600 * 24)] #\n",
        "df_prediction['expdate'] = df_prediction['expdate'].apply(lambda x:time.strftime(\"%Y-%m-%d\", time.localtime(x))) \n",
        "df_prediction.tail()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "C:\\Users\\user\\Anaconda3\\envs\\tf2gpu\\lib\\site-packages\\ipykernel_launcher.py:13: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  del sys.path[0]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>year</th>\n",
              "      <th>month</th>\n",
              "      <th>day</th>\n",
              "      <th>hour</th>\n",
              "      <th>expdate</th>\n",
              "      <th>tau</th>\n",
              "      <th>stationID</th>\n",
              "      <th>calPMf</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>595987</th>\n",
              "      <td>2019</td>\n",
              "      <td>11</td>\n",
              "      <td>4</td>\n",
              "      <td>23</td>\n",
              "      <td>2019-10-31</td>\n",
              "      <td>96</td>\n",
              "      <td>EPA084</td>\n",
              "      <td>0.159642</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>595988</th>\n",
              "      <td>2019</td>\n",
              "      <td>11</td>\n",
              "      <td>4</td>\n",
              "      <td>23</td>\n",
              "      <td>2019-10-31</td>\n",
              "      <td>96</td>\n",
              "      <td>EPA101</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>595989</th>\n",
              "      <td>2019</td>\n",
              "      <td>11</td>\n",
              "      <td>4</td>\n",
              "      <td>23</td>\n",
              "      <td>2019-10-31</td>\n",
              "      <td>96</td>\n",
              "      <td>EPA102</td>\n",
              "      <td>0.250123</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>595990</th>\n",
              "      <td>2019</td>\n",
              "      <td>11</td>\n",
              "      <td>4</td>\n",
              "      <td>23</td>\n",
              "      <td>2019-10-31</td>\n",
              "      <td>96</td>\n",
              "      <td>EPA104</td>\n",
              "      <td>0.521565</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>595991</th>\n",
              "      <td>2019</td>\n",
              "      <td>11</td>\n",
              "      <td>4</td>\n",
              "      <td>23</td>\n",
              "      <td>2019-10-31</td>\n",
              "      <td>96</td>\n",
              "      <td>EPA110</td>\n",
              "      <td>18.928040</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        year  month  day  hour     expdate  tau stationID     calPMf\n",
              "595987  2019     11    4    23  2019-10-31   96    EPA084   0.159642\n",
              "595988  2019     11    4    23  2019-10-31   96    EPA101        NaN\n",
              "595989  2019     11    4    23  2019-10-31   96    EPA102   0.250123\n",
              "595990  2019     11    4    23  2019-10-31   96    EPA104   0.521565\n",
              "595991  2019     11    4    23  2019-10-31   96    EPA110  18.928040"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mi_bpTQ0bpYp"
      },
      "source": [
        "## 拿取預測值、測站移除"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hcgYvtnPbpYq",
        "outputId": "13500ae0-9e9f-495c-a8a9-d9e66eebb639"
      },
      "source": [
        "df_prediction = df_prediction[df_prediction['tau']!=0]\n",
        "df_prediction = df_prediction[df_prediction['stationID']!='EPA101']\n",
        "df_prediction = df_prediction[df_prediction['stationID']!='EPA084']\n",
        "df_prediction = df_prediction[df_prediction['stationID']!='EPA104']\n",
        "df_prediction.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>year</th>\n",
              "      <th>month</th>\n",
              "      <th>day</th>\n",
              "      <th>hour</th>\n",
              "      <th>expdate</th>\n",
              "      <th>tau</th>\n",
              "      <th>stationID</th>\n",
              "      <th>calPMf</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>297240</th>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>30</td>\n",
              "      <td>0</td>\n",
              "      <td>2019-09-29</td>\n",
              "      <td>1</td>\n",
              "      <td>EPA001</td>\n",
              "      <td>10.384020</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>297241</th>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>30</td>\n",
              "      <td>0</td>\n",
              "      <td>2019-09-29</td>\n",
              "      <td>1</td>\n",
              "      <td>EPA002</td>\n",
              "      <td>12.222376</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>297242</th>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>30</td>\n",
              "      <td>0</td>\n",
              "      <td>2019-09-29</td>\n",
              "      <td>1</td>\n",
              "      <td>EPA003</td>\n",
              "      <td>5.971256</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>297243</th>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>30</td>\n",
              "      <td>0</td>\n",
              "      <td>2019-09-29</td>\n",
              "      <td>1</td>\n",
              "      <td>EPA004</td>\n",
              "      <td>11.441598</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>297244</th>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>30</td>\n",
              "      <td>0</td>\n",
              "      <td>2019-09-29</td>\n",
              "      <td>1</td>\n",
              "      <td>EPA005</td>\n",
              "      <td>10.245286</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        year  month  day  hour     expdate  tau stationID     calPMf\n",
              "297240  2019      9   30     0  2019-09-29    1    EPA001  10.384020\n",
              "297241  2019      9   30     0  2019-09-29    1    EPA002  12.222376\n",
              "297242  2019      9   30     0  2019-09-29    1    EPA003   5.971256\n",
              "297243  2019      9   30     0  2019-09-29    1    EPA004  11.441598\n",
              "297244  2019      9   30     0  2019-09-29    1    EPA005  10.245286"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bCWLc0B6bpYr"
      },
      "source": [
        "## dateTime為預測的時間"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "04B4UsrxbpYs",
        "outputId": "18457e95-643b-4800-89ec-c1f744299f7e"
      },
      "source": [
        "df_prediction['expdate'] = pd.to_datetime(df_prediction['expdate']).apply(lambda x:int(time.mktime(x.timetuple())))\n",
        "df_prediction['expdate'] = df_prediction['expdate'].apply(lambda x:time.strftime(\"%Y-%m-%d\",time.localtime(x))) \n",
        "df_prediction.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>year</th>\n",
              "      <th>month</th>\n",
              "      <th>day</th>\n",
              "      <th>hour</th>\n",
              "      <th>expdate</th>\n",
              "      <th>tau</th>\n",
              "      <th>stationID</th>\n",
              "      <th>calPMf</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>297240</th>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>30</td>\n",
              "      <td>0</td>\n",
              "      <td>2019-09-29</td>\n",
              "      <td>1</td>\n",
              "      <td>EPA001</td>\n",
              "      <td>10.384020</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>297241</th>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>30</td>\n",
              "      <td>0</td>\n",
              "      <td>2019-09-29</td>\n",
              "      <td>1</td>\n",
              "      <td>EPA002</td>\n",
              "      <td>12.222376</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>297242</th>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>30</td>\n",
              "      <td>0</td>\n",
              "      <td>2019-09-29</td>\n",
              "      <td>1</td>\n",
              "      <td>EPA003</td>\n",
              "      <td>5.971256</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>297243</th>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>30</td>\n",
              "      <td>0</td>\n",
              "      <td>2019-09-29</td>\n",
              "      <td>1</td>\n",
              "      <td>EPA004</td>\n",
              "      <td>11.441598</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>297244</th>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>30</td>\n",
              "      <td>0</td>\n",
              "      <td>2019-09-29</td>\n",
              "      <td>1</td>\n",
              "      <td>EPA005</td>\n",
              "      <td>10.245286</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "        year  month  day  hour     expdate  tau stationID     calPMf\n",
              "297240  2019      9   30     0  2019-09-29    1    EPA001  10.384020\n",
              "297241  2019      9   30     0  2019-09-29    1    EPA002  12.222376\n",
              "297242  2019      9   30     0  2019-09-29    1    EPA003   5.971256\n",
              "297243  2019      9   30     0  2019-09-29    1    EPA004  11.441598\n",
              "297244  2019      9   30     0  2019-09-29    1    EPA005  10.245286"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HujL2gkxbpYt"
      },
      "source": [
        "## 日期、時間缺漏值進行填補"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v_cZ0kIMbpYu"
      },
      "source": [
        "def otherStyleTime(timestamp):\n",
        "    timeArray = time.localtime(timestamp)\n",
        "    date = time.strftime(\"%Y-%m-%d\", timeArray)\n",
        "    return(date)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cbIGMLbnbpYv"
      },
      "source": [
        "start_pre = otherStyleTime(start)\n",
        "end_pre = otherStyleTime(end)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x1GWaqtdbpYx",
        "outputId": "c260ff86-ff54-4cbc-c268-4b8d27008edf"
      },
      "source": [
        "stations = df_prediction['stationID'].unique()\n",
        "all_expdate = getEveryDay(start_pre, end_pre)\n",
        "print('------prediction 補缺失日期開始------')\n",
        "for name in stations:\n",
        "    \n",
        "    # 補一整天皆缺失的值\n",
        "    unnormal = list(df_prediction[df_prediction['stationID'] == name]['expdate'])\n",
        "    dates = list(set(all_expdate).difference(set(unnormal)))\n",
        "\n",
        "    for date in dates:\n",
        "        df_add = df_prediction[: time_step].copy()\n",
        "        df_add['expdate'] = date\n",
        "        df_add['hour'] = list(np.arange(24))*time_interval\n",
        "        df_add['stationID'] = name\n",
        "        df_add['tau'] = np.arange(time_step) + 1\n",
        "        \n",
        "        day = []\n",
        "        month = []\n",
        "        year = []\n",
        "        for t in range(1, time_interval+1):\n",
        "            day += [(pd.to_datetime(date) + timedelta(days=t)).day] * 24 \n",
        "            month += [(pd.to_datetime(date) + timedelta(days=t)).month] * 24 \n",
        "            year += [(pd.to_datetime(date) + timedelta(days=t)).year] * 24 \n",
        "        df_add['day'] = day\n",
        "        df_add['month'] = month\n",
        "        df_add['year'] = year\n",
        "        df_add['calPMf'] = np.nan\n",
        "        df_prediction = pd.concat([df_prediction, df_add])\n",
        "\n",
        "    df_prediction.sort_values(by=['expdate', 'tau', 'stationID'], inplace = True)\n",
        "\n",
        "    # 補部分缺失的值\n",
        "    if sum(pd.value_counts(df_prediction[df_prediction['stationID']==name]['expdate'])!=time_step)>0:\n",
        "        #print(name, date)\n",
        "        count = pd.value_counts(df_prediction[df_prediction['stationID']==name]['expdate']).values\n",
        "        index = pd.value_counts(df_prediction[df_prediction['stationID']==name]['expdate']).index\n",
        "        for expdate in index[count!=time_step]:\n",
        "            all_hour = np.arange(time_step) + 1\n",
        "            unnormal = df_prediction[df_prediction['stationID']==name][df_prediction['expdate']==expdate]['tau']\n",
        "            taus = list(set(all_hour).difference(set(unnormal)))\n",
        "            num = len(hours)\n",
        "\n",
        "            df_add = df_prediction[: len(taus)].copy()\n",
        "            df_add['expdate'] = expdate  \n",
        "            df_add['hour'] = taus % 24 \n",
        "            df_add['stationID'] = name\n",
        "            df_add['tau'] = taus\n",
        "            df_add['day'] = (pd.to_datetime(date) + timedelta(hours=23+taus)).day\n",
        "            df_add['month'] = (pd.to_datetime(date) + timedelta(hours=23+taus)).month\n",
        "            df_add['year'] = (pd.to_datetime(date) + timedelta(hours=23+taus)).year\n",
        "            df_add['calPMf'] = np.nan\n",
        "            df_prediction = pd.concat([df_prediction, df_add])\n",
        "\n",
        "    df_prediction.sort_values(by=['expdate', 'tau', 'stationID'], inplace = True)\n",
        "    print('stations:', name)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "------prediction 補缺失日期開始------\n",
            "stations: EPA001\n",
            "stations: EPA002\n",
            "stations: EPA003\n",
            "stations: EPA004\n",
            "stations: EPA005\n",
            "stations: EPA006\n",
            "stations: EPA007\n",
            "stations: EPA008\n",
            "stations: EPA009\n",
            "stations: EPA010\n",
            "stations: EPA011\n",
            "stations: EPA012\n",
            "stations: EPA013\n",
            "stations: EPA014\n",
            "stations: EPA015\n",
            "stations: EPA016\n",
            "stations: EPA017\n",
            "stations: EPA018\n",
            "stations: EPA019\n",
            "stations: EPA020\n",
            "stations: EPA021\n",
            "stations: EPA022\n",
            "stations: EPA023\n",
            "stations: EPA024\n",
            "stations: EPA025\n",
            "stations: EPA026\n",
            "stations: EPA027\n",
            "stations: EPA028\n",
            "stations: EPA029\n",
            "stations: EPA030\n",
            "stations: EPA031\n",
            "stations: EPA032\n",
            "stations: EPA033\n",
            "stations: EPA034\n",
            "stations: EPA035\n",
            "stations: EPA036\n",
            "stations: EPA037\n",
            "stations: EPA038\n",
            "stations: EPA039\n",
            "stations: EPA040\n",
            "stations: EPA041\n",
            "stations: EPA042\n",
            "stations: EPA043\n",
            "stations: EPA044\n",
            "stations: EPA045\n",
            "stations: EPA046\n",
            "stations: EPA047\n",
            "stations: EPA048\n",
            "stations: EPA049\n",
            "stations: EPA050\n",
            "stations: EPA051\n",
            "stations: EPA052\n",
            "stations: EPA053\n",
            "stations: EPA054\n",
            "stations: EPA056\n",
            "stations: EPA057\n",
            "stations: EPA058\n",
            "stations: EPA059\n",
            "stations: EPA060\n",
            "stations: EPA061\n",
            "stations: EPA062\n",
            "stations: EPA063\n",
            "stations: EPA064\n",
            "stations: EPA065\n",
            "stations: EPA066\n",
            "stations: EPA067\n",
            "stations: EPA068\n",
            "stations: EPA069\n",
            "stations: EPA070\n",
            "stations: EPA071\n",
            "stations: EPA072\n",
            "stations: EPA080\n",
            "stations: EPA102\n",
            "stations: EPA110\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EOKIZP19ni7q",
        "outputId": "240a8052-5291-4c82-8ac5-282d77367c35"
      },
      "source": [
        "if len(np.unique(pd.value_counts(df_prediction['stationID'])))==1:\n",
        "    print(\"prediction補缺失日期成功\")\n",
        "else:\n",
        "    print(\"prediction補缺失日期失敗\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "prediction補缺失日期成功\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "apFH93M3bpYz",
        "outputId": "d00c7418-b351-46e8-d522-d9e6cb19836c"
      },
      "source": [
        "df_prediction = df_prediction[['expdate','stationID','year','month','day','hour','tau','calPMf']]\n",
        "df_prediction.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>expdate</th>\n",
              "      <th>stationID</th>\n",
              "      <th>year</th>\n",
              "      <th>month</th>\n",
              "      <th>day</th>\n",
              "      <th>hour</th>\n",
              "      <th>tau</th>\n",
              "      <th>calPMf</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1932288</td>\n",
              "      <td>2019-09-29</td>\n",
              "      <td>EPA001</td>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>30</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>10.384020</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1932289</td>\n",
              "      <td>2019-09-29</td>\n",
              "      <td>EPA002</td>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>30</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>12.222376</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1932290</td>\n",
              "      <td>2019-09-29</td>\n",
              "      <td>EPA003</td>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>30</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>5.971256</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1932291</td>\n",
              "      <td>2019-09-29</td>\n",
              "      <td>EPA004</td>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>30</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>11.441598</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1932292</td>\n",
              "      <td>2019-09-29</td>\n",
              "      <td>EPA005</td>\n",
              "      <td>2019</td>\n",
              "      <td>9</td>\n",
              "      <td>30</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>10.245286</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            expdate stationID  year  month  day  hour  tau     calPMf\n",
              "1932288  2019-09-29    EPA001  2019      9   30     0    1  10.384020\n",
              "1932289  2019-09-29    EPA002  2019      9   30     0    1  12.222376\n",
              "1932290  2019-09-29    EPA003  2019      9   30     0    1   5.971256\n",
              "1932291  2019-09-29    EPA004  2019      9   30     0    1  11.441598\n",
              "1932292  2019-09-29    EPA005  2019      9   30     0    1  10.245286"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a8R4pbx1cVvT"
      },
      "source": [
        "df_prediction.to_csv('old_corr_preciction.txt', sep='\\t', index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RrZLh92AbpY0"
      },
      "source": [
        "df_prediction.to_csv('MissingValue_Prediction.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T3PaGA6cbpY3"
      },
      "source": [
        "## 將tau = 1的日期設為index"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iQvKWr-abpY5",
        "outputId": "e810ea5c-4d48-4f6b-9434-b38c400272be"
      },
      "source": [
        "df_pred = pd.DataFrame(np.array(df_prediction['calPMf']).reshape([-1, time_step*station_num]))\n",
        "df_pred.index = pd.to_datetime(df_prediction['expdate'].unique(), errors='coerce') + timedelta(days=1)\n",
        "index = df_pred.index \n",
        "df_pred.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>...</th>\n",
              "      <th>7094</th>\n",
              "      <th>7095</th>\n",
              "      <th>7096</th>\n",
              "      <th>7097</th>\n",
              "      <th>7098</th>\n",
              "      <th>7099</th>\n",
              "      <th>7100</th>\n",
              "      <th>7101</th>\n",
              "      <th>7102</th>\n",
              "      <th>7103</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>2019-09-30</td>\n",
              "      <td>10.384020</td>\n",
              "      <td>12.222376</td>\n",
              "      <td>5.971256</td>\n",
              "      <td>11.441598</td>\n",
              "      <td>10.245286</td>\n",
              "      <td>14.770390</td>\n",
              "      <td>14.752220</td>\n",
              "      <td>16.288942</td>\n",
              "      <td>15.054422</td>\n",
              "      <td>9.530121</td>\n",
              "      <td>...</td>\n",
              "      <td>3.265403</td>\n",
              "      <td>4.607365</td>\n",
              "      <td>5.798550</td>\n",
              "      <td>20.407620</td>\n",
              "      <td>2.930961</td>\n",
              "      <td>108.79449</td>\n",
              "      <td>2.566029</td>\n",
              "      <td>4.219114</td>\n",
              "      <td>NaN</td>\n",
              "      <td>5.080588</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-01</td>\n",
              "      <td>2.303247</td>\n",
              "      <td>1.167996</td>\n",
              "      <td>3.119260</td>\n",
              "      <td>1.364694</td>\n",
              "      <td>2.355424</td>\n",
              "      <td>3.042923</td>\n",
              "      <td>3.324206</td>\n",
              "      <td>9.170274</td>\n",
              "      <td>2.758917</td>\n",
              "      <td>1.507425</td>\n",
              "      <td>...</td>\n",
              "      <td>6.049601</td>\n",
              "      <td>10.028864</td>\n",
              "      <td>19.171390</td>\n",
              "      <td>36.775890</td>\n",
              "      <td>10.900753</td>\n",
              "      <td>19.79402</td>\n",
              "      <td>13.954580</td>\n",
              "      <td>0.923192</td>\n",
              "      <td>1.547703</td>\n",
              "      <td>18.794590</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-02</td>\n",
              "      <td>18.529622</td>\n",
              "      <td>16.354433</td>\n",
              "      <td>18.980730</td>\n",
              "      <td>14.328082</td>\n",
              "      <td>18.858808</td>\n",
              "      <td>20.995500</td>\n",
              "      <td>22.920832</td>\n",
              "      <td>18.492727</td>\n",
              "      <td>28.264308</td>\n",
              "      <td>17.527233</td>\n",
              "      <td>...</td>\n",
              "      <td>6.335702</td>\n",
              "      <td>20.700720</td>\n",
              "      <td>29.276700</td>\n",
              "      <td>5.626096</td>\n",
              "      <td>15.481800</td>\n",
              "      <td>26.24797</td>\n",
              "      <td>2.623062</td>\n",
              "      <td>0.400118</td>\n",
              "      <td>24.339509</td>\n",
              "      <td>45.498040</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-03</td>\n",
              "      <td>25.217936</td>\n",
              "      <td>27.691776</td>\n",
              "      <td>26.745528</td>\n",
              "      <td>24.618408</td>\n",
              "      <td>33.900899</td>\n",
              "      <td>35.589010</td>\n",
              "      <td>33.690254</td>\n",
              "      <td>38.341235</td>\n",
              "      <td>34.809658</td>\n",
              "      <td>29.657188</td>\n",
              "      <td>...</td>\n",
              "      <td>2.452619</td>\n",
              "      <td>57.572800</td>\n",
              "      <td>4.243239</td>\n",
              "      <td>1.428393</td>\n",
              "      <td>35.264520</td>\n",
              "      <td>67.52889</td>\n",
              "      <td>2.677509</td>\n",
              "      <td>0.316854</td>\n",
              "      <td>208.070305</td>\n",
              "      <td>15.228980</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-04</td>\n",
              "      <td>38.854236</td>\n",
              "      <td>36.755066</td>\n",
              "      <td>35.572574</td>\n",
              "      <td>34.110252</td>\n",
              "      <td>48.683100</td>\n",
              "      <td>52.354600</td>\n",
              "      <td>49.098145</td>\n",
              "      <td>44.835977</td>\n",
              "      <td>43.202534</td>\n",
              "      <td>41.142932</td>\n",
              "      <td>...</td>\n",
              "      <td>5.044891</td>\n",
              "      <td>8.853860</td>\n",
              "      <td>7.402682</td>\n",
              "      <td>2.748726</td>\n",
              "      <td>9.494940</td>\n",
              "      <td>52.39232</td>\n",
              "      <td>4.041953</td>\n",
              "      <td>0.718312</td>\n",
              "      <td>55.862532</td>\n",
              "      <td>41.004660</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 7104 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                 0          1          2          3          4          5     \\\n",
              "2019-09-30  10.384020  12.222376   5.971256  11.441598  10.245286  14.770390   \n",
              "2019-10-01   2.303247   1.167996   3.119260   1.364694   2.355424   3.042923   \n",
              "2019-10-02  18.529622  16.354433  18.980730  14.328082  18.858808  20.995500   \n",
              "2019-10-03  25.217936  27.691776  26.745528  24.618408  33.900899  35.589010   \n",
              "2019-10-04  38.854236  36.755066  35.572574  34.110252  48.683100  52.354600   \n",
              "\n",
              "                 6          7          8          9     ...      7094  \\\n",
              "2019-09-30  14.752220  16.288942  15.054422   9.530121  ...  3.265403   \n",
              "2019-10-01   3.324206   9.170274   2.758917   1.507425  ...  6.049601   \n",
              "2019-10-02  22.920832  18.492727  28.264308  17.527233  ...  6.335702   \n",
              "2019-10-03  33.690254  38.341235  34.809658  29.657188  ...  2.452619   \n",
              "2019-10-04  49.098145  44.835977  43.202534  41.142932  ...  5.044891   \n",
              "\n",
              "                 7095       7096       7097       7098       7099       7100  \\\n",
              "2019-09-30   4.607365   5.798550  20.407620   2.930961  108.79449   2.566029   \n",
              "2019-10-01  10.028864  19.171390  36.775890  10.900753   19.79402  13.954580   \n",
              "2019-10-02  20.700720  29.276700   5.626096  15.481800   26.24797   2.623062   \n",
              "2019-10-03  57.572800   4.243239   1.428393  35.264520   67.52889   2.677509   \n",
              "2019-10-04   8.853860   7.402682   2.748726   9.494940   52.39232   4.041953   \n",
              "\n",
              "                7101        7102       7103  \n",
              "2019-09-30  4.219114         NaN   5.080588  \n",
              "2019-10-01  0.923192    1.547703  18.794590  \n",
              "2019-10-02  0.400118   24.339509  45.498040  \n",
              "2019-10-03  0.316854  208.070305  15.228980  \n",
              "2019-10-04  0.718312   55.862532  41.004660  \n",
              "\n",
              "[5 rows x 7104 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KvIT6TW1bpY7"
      },
      "source": [
        "## 將一整天為NAN的值刪掉"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BeYV4XDJbpY7",
        "outputId": "be92a513-0311-44c3-f7c4-72c2b84cebe1"
      },
      "source": [
        "df_pred = df_pred.dropna(axis=0, how='all')\n",
        "index = df_pred.index\n",
        "df_pred.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>...</th>\n",
              "      <th>7094</th>\n",
              "      <th>7095</th>\n",
              "      <th>7096</th>\n",
              "      <th>7097</th>\n",
              "      <th>7098</th>\n",
              "      <th>7099</th>\n",
              "      <th>7100</th>\n",
              "      <th>7101</th>\n",
              "      <th>7102</th>\n",
              "      <th>7103</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>2019-09-30</td>\n",
              "      <td>10.384020</td>\n",
              "      <td>12.222376</td>\n",
              "      <td>5.971256</td>\n",
              "      <td>11.441598</td>\n",
              "      <td>10.245286</td>\n",
              "      <td>14.770390</td>\n",
              "      <td>14.752220</td>\n",
              "      <td>16.288942</td>\n",
              "      <td>15.054422</td>\n",
              "      <td>9.530121</td>\n",
              "      <td>...</td>\n",
              "      <td>3.265403</td>\n",
              "      <td>4.607365</td>\n",
              "      <td>5.798550</td>\n",
              "      <td>20.407620</td>\n",
              "      <td>2.930961</td>\n",
              "      <td>108.79449</td>\n",
              "      <td>2.566029</td>\n",
              "      <td>4.219114</td>\n",
              "      <td>NaN</td>\n",
              "      <td>5.080588</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-01</td>\n",
              "      <td>2.303247</td>\n",
              "      <td>1.167996</td>\n",
              "      <td>3.119260</td>\n",
              "      <td>1.364694</td>\n",
              "      <td>2.355424</td>\n",
              "      <td>3.042923</td>\n",
              "      <td>3.324206</td>\n",
              "      <td>9.170274</td>\n",
              "      <td>2.758917</td>\n",
              "      <td>1.507425</td>\n",
              "      <td>...</td>\n",
              "      <td>6.049601</td>\n",
              "      <td>10.028864</td>\n",
              "      <td>19.171390</td>\n",
              "      <td>36.775890</td>\n",
              "      <td>10.900753</td>\n",
              "      <td>19.79402</td>\n",
              "      <td>13.954580</td>\n",
              "      <td>0.923192</td>\n",
              "      <td>1.547703</td>\n",
              "      <td>18.794590</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-02</td>\n",
              "      <td>18.529622</td>\n",
              "      <td>16.354433</td>\n",
              "      <td>18.980730</td>\n",
              "      <td>14.328082</td>\n",
              "      <td>18.858808</td>\n",
              "      <td>20.995500</td>\n",
              "      <td>22.920832</td>\n",
              "      <td>18.492727</td>\n",
              "      <td>28.264308</td>\n",
              "      <td>17.527233</td>\n",
              "      <td>...</td>\n",
              "      <td>6.335702</td>\n",
              "      <td>20.700720</td>\n",
              "      <td>29.276700</td>\n",
              "      <td>5.626096</td>\n",
              "      <td>15.481800</td>\n",
              "      <td>26.24797</td>\n",
              "      <td>2.623062</td>\n",
              "      <td>0.400118</td>\n",
              "      <td>24.339509</td>\n",
              "      <td>45.498040</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-03</td>\n",
              "      <td>25.217936</td>\n",
              "      <td>27.691776</td>\n",
              "      <td>26.745528</td>\n",
              "      <td>24.618408</td>\n",
              "      <td>33.900899</td>\n",
              "      <td>35.589010</td>\n",
              "      <td>33.690254</td>\n",
              "      <td>38.341235</td>\n",
              "      <td>34.809658</td>\n",
              "      <td>29.657188</td>\n",
              "      <td>...</td>\n",
              "      <td>2.452619</td>\n",
              "      <td>57.572800</td>\n",
              "      <td>4.243239</td>\n",
              "      <td>1.428393</td>\n",
              "      <td>35.264520</td>\n",
              "      <td>67.52889</td>\n",
              "      <td>2.677509</td>\n",
              "      <td>0.316854</td>\n",
              "      <td>208.070305</td>\n",
              "      <td>15.228980</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-04</td>\n",
              "      <td>38.854236</td>\n",
              "      <td>36.755066</td>\n",
              "      <td>35.572574</td>\n",
              "      <td>34.110252</td>\n",
              "      <td>48.683100</td>\n",
              "      <td>52.354600</td>\n",
              "      <td>49.098145</td>\n",
              "      <td>44.835977</td>\n",
              "      <td>43.202534</td>\n",
              "      <td>41.142932</td>\n",
              "      <td>...</td>\n",
              "      <td>5.044891</td>\n",
              "      <td>8.853860</td>\n",
              "      <td>7.402682</td>\n",
              "      <td>2.748726</td>\n",
              "      <td>9.494940</td>\n",
              "      <td>52.39232</td>\n",
              "      <td>4.041953</td>\n",
              "      <td>0.718312</td>\n",
              "      <td>55.862532</td>\n",
              "      <td>41.004660</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 7104 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                 0          1          2          3          4          5     \\\n",
              "2019-09-30  10.384020  12.222376   5.971256  11.441598  10.245286  14.770390   \n",
              "2019-10-01   2.303247   1.167996   3.119260   1.364694   2.355424   3.042923   \n",
              "2019-10-02  18.529622  16.354433  18.980730  14.328082  18.858808  20.995500   \n",
              "2019-10-03  25.217936  27.691776  26.745528  24.618408  33.900899  35.589010   \n",
              "2019-10-04  38.854236  36.755066  35.572574  34.110252  48.683100  52.354600   \n",
              "\n",
              "                 6          7          8          9     ...      7094  \\\n",
              "2019-09-30  14.752220  16.288942  15.054422   9.530121  ...  3.265403   \n",
              "2019-10-01   3.324206   9.170274   2.758917   1.507425  ...  6.049601   \n",
              "2019-10-02  22.920832  18.492727  28.264308  17.527233  ...  6.335702   \n",
              "2019-10-03  33.690254  38.341235  34.809658  29.657188  ...  2.452619   \n",
              "2019-10-04  49.098145  44.835977  43.202534  41.142932  ...  5.044891   \n",
              "\n",
              "                 7095       7096       7097       7098       7099       7100  \\\n",
              "2019-09-30   4.607365   5.798550  20.407620   2.930961  108.79449   2.566029   \n",
              "2019-10-01  10.028864  19.171390  36.775890  10.900753   19.79402  13.954580   \n",
              "2019-10-02  20.700720  29.276700   5.626096  15.481800   26.24797   2.623062   \n",
              "2019-10-03  57.572800   4.243239   1.428393  35.264520   67.52889   2.677509   \n",
              "2019-10-04   8.853860   7.402682   2.748726   9.494940   52.39232   4.041953   \n",
              "\n",
              "                7101        7102       7103  \n",
              "2019-09-30  4.219114         NaN   5.080588  \n",
              "2019-10-01  0.923192    1.547703  18.794590  \n",
              "2019-10-02  0.400118   24.339509  45.498040  \n",
              "2019-10-03  0.316854  208.070305  15.228980  \n",
              "2019-10-04  0.718312   55.862532  41.004660  \n",
              "\n",
              "[5 rows x 7104 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wccZaAz6bpY9"
      },
      "source": [
        "## 將PMF=NAN進行補值"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kTra-kQkbpY-",
        "outputId": "807cbc94-f76c-4582-cca4-759eeb27244f"
      },
      "source": [
        "rpy2_new_prediction = pandas2ri.py2ri(df_pred)\n",
        "robjects.r.assign('rpy2_new_prediction', rpy2_new_prediction)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "\n",
              "    <span>R/rpy2 DataFrame (33 x 7104)</span>\n",
              "    <table>\n",
              "      <thead>\n",
              "        <tr>\n",
              "        \n",
              "          <th>X0</th>\n",
              "        \n",
              "          <th>X1</th>\n",
              "        \n",
              "          <th>X2</th>\n",
              "        \n",
              "          <th>...</th>\n",
              "        \n",
              "          <th>X7101</th>\n",
              "        \n",
              "          <th>X7102</th>\n",
              "        \n",
              "          <th>X7103</th>\n",
              "        \n",
              "        </tr>\n",
              "      </thead>\n",
              "      <tbody>\n",
              "      \n",
              "      <tr>\n",
              "      \n",
              "      <td>\n",
              "        10.384020\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        12.222376\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        5.971256\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        4.219114\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "             nan\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        5.080588\n",
              "      </td>\n",
              "      \n",
              "      </tr>\n",
              "      \n",
              "      <tr>\n",
              "      \n",
              "      <td>\n",
              "        2.303247\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        1.167996\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        3.119260\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        0.923192\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        1.547703\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        18.794590\n",
              "      </td>\n",
              "      \n",
              "      </tr>\n",
              "      \n",
              "      <tr>\n",
              "      \n",
              "      <td>\n",
              "        18.529622\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        16.354433\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        18.980730\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        0.400118\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        24.339509\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        45.498040\n",
              "      </td>\n",
              "      \n",
              "      </tr>\n",
              "      \n",
              "      <tr>\n",
              "      \n",
              "      <td>\n",
              "        25.217936\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        27.691776\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        26.745528\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        0.316854\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        208.070305\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        15.228980\n",
              "      </td>\n",
              "      \n",
              "      </tr>\n",
              "      \n",
              "      <tr>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      </tr>\n",
              "      \n",
              "      <tr>\n",
              "      \n",
              "      <td>\n",
              "        5.632875\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        4.230715\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        0.269621\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        1.321762\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        1.656864\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        6.052550\n",
              "      </td>\n",
              "      \n",
              "      </tr>\n",
              "      \n",
              "      <tr>\n",
              "      \n",
              "      <td>\n",
              "        1.289931\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        19.258952\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        0.432383\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        0.457557\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        0.256255\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        12.734080\n",
              "      </td>\n",
              "      \n",
              "      </tr>\n",
              "      \n",
              "      <tr>\n",
              "      \n",
              "      <td>\n",
              "        1.264438\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        29.788514\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        0.255462\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        0.760478\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        0.268489\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        10.029330\n",
              "      </td>\n",
              "      \n",
              "      </tr>\n",
              "      \n",
              "      <tr>\n",
              "      \n",
              "      <td>\n",
              "        2.259185\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        19.170722\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        0.316616\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        ...\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        0.492021\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        0.250123\n",
              "      </td>\n",
              "      \n",
              "      <td>\n",
              "        18.928040\n",
              "      </td>\n",
              "      \n",
              "      </tr>\n",
              "      \n",
              "      </tbody>\n",
              "    </table>\n",
              "    "
            ],
            "text/plain": [
              "R object with classes: ('data.frame',) mapped to:\n",
              "<DataFrame - Python:0x0000021793FA0C08 / R:0x00000217BEDCE7C0>\n",
              "[FloatVector, FloatVector, FloatVector, FloatVector, ..., FloatVector, FloatVector, FloatVector, FloatVector]\n",
              "  X0: <class 'rpy2.robjects.vectors.FloatVector'>\n",
              "  R object with classes: ('numeric',) mapped to:\n",
              "<FloatVector - Python:0x0000021793FA0F48 / R:0x00000217AFC4FCE8>\n",
              "[10.384020, 2.303247, 18.529622, 25.217936, ..., 5.632875, 1.289931, 1.264438, 2.259185]\n",
              "  X1: <class 'rpy2.robjects.vectors.FloatVector'>\n",
              "  R object with classes: ('numeric',) mapped to:\n",
              "<FloatVector - Python:0x0000021793FA0908 / R:0x00000217AFC4FE28>\n",
              "[12.222376, 1.167996, 16.354433, 27.691776, ..., 4.230715, 19.258952, 29.788514, 19.170722]\n",
              "  X2: <class 'rpy2.robjects.vectors.FloatVector'>\n",
              "  R object with classes: ('numeric',) mapped to:\n",
              "<FloatVector - Python:0x0000021793FA0608 / R:0x00000217AB93F910>\n",
              "[5.971256, 3.119260, 18.980730, 26.745528, ..., 0.269621, 0.432383, 0.255462, 0.316616]\n",
              "  X3: <class 'rpy2.robjects.vectors.FloatVector'>\n",
              "  R object with classes: ('numeric',) mapped to:\n",
              "<FloatVector - Python:0x0000021793FA0208 / R:0x00000217AB93FC88>\n",
              "[11.441598, 1.364694, 14.328082, 24.618408, ..., 2.059012, 14.901676, 22.720576, 20.142869]\n",
              "...\n",
              "  X5: <class 'rpy2.robjects.vectors.FloatVector'>\n",
              "  R object with classes: ('numeric',) mapped to:\n",
              "<FloatVector - Python:0x0000021793F96948 / R:0x00000217955526F0>\n",
              "[2.566029, 13.954580, 2.623062, 2.677509, ..., 5.899132, 32.502360, 47.719180, 25.398300]\n",
              "  X6: <class 'rpy2.robjects.vectors.FloatVector'>\n",
              "  R object with classes: ('numeric',) mapped to:\n",
              "<FloatVector - Python:0x0000021793F85EC8 / R:0x0000021795552830>\n",
              "[4.219114, 0.923192, 0.400118, 0.316854, ..., 1.321762, 0.457557, 0.760478, 0.492021]\n",
              "  X7: <class 'rpy2.robjects.vectors.FloatVector'>\n",
              "  R object with classes: ('numeric',) mapped to:\n",
              "<FloatVector - Python:0x0000021793F8EB08 / R:0x000002179555A3F0>\n",
              "[     nan, 1.547703, 24.339509, 208.070305, ..., 1.656864, 0.256255, 0.268489, 0.250123]\n",
              "  X8: <class 'rpy2.robjects.vectors.FloatVector'>\n",
              "  R object with classes: ('numeric',) mapped to:\n",
              "<FloatVector - Python:0x000002179403A0C8 / R:0x000002179555A530>\n",
              "[5.080588, 18.794590, 45.498040, 15.228980, ..., 6.052550, 12.734080, 10.029330, 18.928040]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n2AZNAGKbpZA",
        "outputId": "d43ff54d-21fb-4964-f0ba-0c35e05fe6a0"
      },
      "source": [
        "r('''\n",
        "rpy2_new_prediction[is.na(rpy2_new_prediction)]<- NaN\n",
        "y <- data.matrix(rpy2_new_prediction)\n",
        "Ya <- dineof(y)$Xa\n",
        "new_prediction<-data.frame(Ya)\n",
        "head(new_prediction)\n",
        "''')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>X0</th>\n",
              "      <th>X1</th>\n",
              "      <th>X2</th>\n",
              "      <th>X3</th>\n",
              "      <th>X4</th>\n",
              "      <th>X5</th>\n",
              "      <th>X6</th>\n",
              "      <th>X7</th>\n",
              "      <th>X8</th>\n",
              "      <th>X9</th>\n",
              "      <th>...</th>\n",
              "      <th>X7094</th>\n",
              "      <th>X7095</th>\n",
              "      <th>X7096</th>\n",
              "      <th>X7097</th>\n",
              "      <th>X7098</th>\n",
              "      <th>X7099</th>\n",
              "      <th>X7100</th>\n",
              "      <th>X7101</th>\n",
              "      <th>X7102</th>\n",
              "      <th>X7103</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>0</td>\n",
              "      <td>10.384020</td>\n",
              "      <td>12.222376</td>\n",
              "      <td>5.971256</td>\n",
              "      <td>11.441598</td>\n",
              "      <td>10.245286</td>\n",
              "      <td>14.770390</td>\n",
              "      <td>14.752220</td>\n",
              "      <td>16.288942</td>\n",
              "      <td>15.054422</td>\n",
              "      <td>9.530121</td>\n",
              "      <td>...</td>\n",
              "      <td>3.265403</td>\n",
              "      <td>4.607365</td>\n",
              "      <td>5.798550</td>\n",
              "      <td>20.407620</td>\n",
              "      <td>2.930961</td>\n",
              "      <td>108.79449</td>\n",
              "      <td>2.566029</td>\n",
              "      <td>4.219114</td>\n",
              "      <td>-6.518932</td>\n",
              "      <td>5.080588</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>2.303247</td>\n",
              "      <td>1.167996</td>\n",
              "      <td>3.119260</td>\n",
              "      <td>1.364694</td>\n",
              "      <td>2.355424</td>\n",
              "      <td>3.042923</td>\n",
              "      <td>3.324206</td>\n",
              "      <td>9.170274</td>\n",
              "      <td>2.758917</td>\n",
              "      <td>1.507425</td>\n",
              "      <td>...</td>\n",
              "      <td>6.049601</td>\n",
              "      <td>10.028864</td>\n",
              "      <td>19.171390</td>\n",
              "      <td>36.775890</td>\n",
              "      <td>10.900753</td>\n",
              "      <td>19.79402</td>\n",
              "      <td>13.954580</td>\n",
              "      <td>0.923192</td>\n",
              "      <td>1.547703</td>\n",
              "      <td>18.794590</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>18.529622</td>\n",
              "      <td>16.354433</td>\n",
              "      <td>18.980730</td>\n",
              "      <td>14.328082</td>\n",
              "      <td>18.858808</td>\n",
              "      <td>20.995500</td>\n",
              "      <td>22.920832</td>\n",
              "      <td>18.492727</td>\n",
              "      <td>28.264308</td>\n",
              "      <td>17.527233</td>\n",
              "      <td>...</td>\n",
              "      <td>6.335702</td>\n",
              "      <td>20.700720</td>\n",
              "      <td>29.276700</td>\n",
              "      <td>5.626096</td>\n",
              "      <td>15.481800</td>\n",
              "      <td>26.24797</td>\n",
              "      <td>2.623062</td>\n",
              "      <td>0.400118</td>\n",
              "      <td>24.339509</td>\n",
              "      <td>45.498040</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>25.217936</td>\n",
              "      <td>27.691776</td>\n",
              "      <td>26.745528</td>\n",
              "      <td>24.618408</td>\n",
              "      <td>33.900899</td>\n",
              "      <td>35.589010</td>\n",
              "      <td>33.690254</td>\n",
              "      <td>38.341235</td>\n",
              "      <td>34.809658</td>\n",
              "      <td>29.657188</td>\n",
              "      <td>...</td>\n",
              "      <td>2.452619</td>\n",
              "      <td>57.572800</td>\n",
              "      <td>4.243239</td>\n",
              "      <td>1.428393</td>\n",
              "      <td>35.264520</td>\n",
              "      <td>67.52889</td>\n",
              "      <td>2.677509</td>\n",
              "      <td>0.316854</td>\n",
              "      <td>208.070305</td>\n",
              "      <td>15.228980</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>38.854236</td>\n",
              "      <td>36.755066</td>\n",
              "      <td>35.572574</td>\n",
              "      <td>34.110252</td>\n",
              "      <td>48.683100</td>\n",
              "      <td>52.354600</td>\n",
              "      <td>49.098145</td>\n",
              "      <td>44.835977</td>\n",
              "      <td>43.202534</td>\n",
              "      <td>41.142932</td>\n",
              "      <td>...</td>\n",
              "      <td>5.044891</td>\n",
              "      <td>8.853860</td>\n",
              "      <td>7.402682</td>\n",
              "      <td>2.748726</td>\n",
              "      <td>9.494940</td>\n",
              "      <td>52.39232</td>\n",
              "      <td>4.041953</td>\n",
              "      <td>0.718312</td>\n",
              "      <td>55.862532</td>\n",
              "      <td>41.004660</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>5</td>\n",
              "      <td>17.153744</td>\n",
              "      <td>17.063243</td>\n",
              "      <td>13.142354</td>\n",
              "      <td>15.819230</td>\n",
              "      <td>24.312800</td>\n",
              "      <td>26.722910</td>\n",
              "      <td>25.678790</td>\n",
              "      <td>22.525031</td>\n",
              "      <td>21.538990</td>\n",
              "      <td>15.539159</td>\n",
              "      <td>...</td>\n",
              "      <td>19.213020</td>\n",
              "      <td>14.916870</td>\n",
              "      <td>9.858000</td>\n",
              "      <td>1.813355</td>\n",
              "      <td>9.422510</td>\n",
              "      <td>49.04423</td>\n",
              "      <td>1.835651</td>\n",
              "      <td>0.650656</td>\n",
              "      <td>33.866711</td>\n",
              "      <td>41.154010</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>6 rows × 7104 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "          X0         X1         X2         X3         X4         X5  \\\n",
              "0  10.384020  12.222376   5.971256  11.441598  10.245286  14.770390   \n",
              "1   2.303247   1.167996   3.119260   1.364694   2.355424   3.042923   \n",
              "2  18.529622  16.354433  18.980730  14.328082  18.858808  20.995500   \n",
              "3  25.217936  27.691776  26.745528  24.618408  33.900899  35.589010   \n",
              "4  38.854236  36.755066  35.572574  34.110252  48.683100  52.354600   \n",
              "5  17.153744  17.063243  13.142354  15.819230  24.312800  26.722910   \n",
              "\n",
              "          X6         X7         X8         X9  ...      X7094      X7095  \\\n",
              "0  14.752220  16.288942  15.054422   9.530121  ...   3.265403   4.607365   \n",
              "1   3.324206   9.170274   2.758917   1.507425  ...   6.049601  10.028864   \n",
              "2  22.920832  18.492727  28.264308  17.527233  ...   6.335702  20.700720   \n",
              "3  33.690254  38.341235  34.809658  29.657188  ...   2.452619  57.572800   \n",
              "4  49.098145  44.835977  43.202534  41.142932  ...   5.044891   8.853860   \n",
              "5  25.678790  22.525031  21.538990  15.539159  ...  19.213020  14.916870   \n",
              "\n",
              "       X7096      X7097      X7098      X7099      X7100     X7101  \\\n",
              "0   5.798550  20.407620   2.930961  108.79449   2.566029  4.219114   \n",
              "1  19.171390  36.775890  10.900753   19.79402  13.954580  0.923192   \n",
              "2  29.276700   5.626096  15.481800   26.24797   2.623062  0.400118   \n",
              "3   4.243239   1.428393  35.264520   67.52889   2.677509  0.316854   \n",
              "4   7.402682   2.748726   9.494940   52.39232   4.041953  0.718312   \n",
              "5   9.858000   1.813355   9.422510   49.04423   1.835651  0.650656   \n",
              "\n",
              "        X7102      X7103  \n",
              "0   -6.518932   5.080588  \n",
              "1    1.547703  18.794590  \n",
              "2   24.339509  45.498040  \n",
              "3  208.070305  15.228980  \n",
              "4   55.862532  41.004660  \n",
              "5   33.866711  41.154010  \n",
              "\n",
              "[6 rows x 7104 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kCXFWIz1bpZC",
        "outputId": "72f7227a-3381-4f19-8442-a4d205bdbdeb"
      },
      "source": [
        "r.data('new_prediction')\n",
        "new_prediction = pd.DataFrame(r['new_prediction'])\n",
        "new_prediction.index = index\n",
        "new_prediction.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>X0</th>\n",
              "      <th>X1</th>\n",
              "      <th>X2</th>\n",
              "      <th>X3</th>\n",
              "      <th>X4</th>\n",
              "      <th>X5</th>\n",
              "      <th>X6</th>\n",
              "      <th>X7</th>\n",
              "      <th>X8</th>\n",
              "      <th>X9</th>\n",
              "      <th>...</th>\n",
              "      <th>X7094</th>\n",
              "      <th>X7095</th>\n",
              "      <th>X7096</th>\n",
              "      <th>X7097</th>\n",
              "      <th>X7098</th>\n",
              "      <th>X7099</th>\n",
              "      <th>X7100</th>\n",
              "      <th>X7101</th>\n",
              "      <th>X7102</th>\n",
              "      <th>X7103</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>2019-09-30</td>\n",
              "      <td>10.384020</td>\n",
              "      <td>12.222376</td>\n",
              "      <td>5.971256</td>\n",
              "      <td>11.441598</td>\n",
              "      <td>10.245286</td>\n",
              "      <td>14.770390</td>\n",
              "      <td>14.752220</td>\n",
              "      <td>16.288942</td>\n",
              "      <td>15.054422</td>\n",
              "      <td>9.530121</td>\n",
              "      <td>...</td>\n",
              "      <td>3.265403</td>\n",
              "      <td>4.607365</td>\n",
              "      <td>5.798550</td>\n",
              "      <td>20.407620</td>\n",
              "      <td>2.930961</td>\n",
              "      <td>108.79449</td>\n",
              "      <td>2.566029</td>\n",
              "      <td>4.219114</td>\n",
              "      <td>-6.518932</td>\n",
              "      <td>5.080588</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-01</td>\n",
              "      <td>2.303247</td>\n",
              "      <td>1.167996</td>\n",
              "      <td>3.119260</td>\n",
              "      <td>1.364694</td>\n",
              "      <td>2.355424</td>\n",
              "      <td>3.042923</td>\n",
              "      <td>3.324206</td>\n",
              "      <td>9.170274</td>\n",
              "      <td>2.758917</td>\n",
              "      <td>1.507425</td>\n",
              "      <td>...</td>\n",
              "      <td>6.049601</td>\n",
              "      <td>10.028864</td>\n",
              "      <td>19.171390</td>\n",
              "      <td>36.775890</td>\n",
              "      <td>10.900753</td>\n",
              "      <td>19.79402</td>\n",
              "      <td>13.954580</td>\n",
              "      <td>0.923192</td>\n",
              "      <td>1.547703</td>\n",
              "      <td>18.794590</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-02</td>\n",
              "      <td>18.529622</td>\n",
              "      <td>16.354433</td>\n",
              "      <td>18.980730</td>\n",
              "      <td>14.328082</td>\n",
              "      <td>18.858808</td>\n",
              "      <td>20.995500</td>\n",
              "      <td>22.920832</td>\n",
              "      <td>18.492727</td>\n",
              "      <td>28.264308</td>\n",
              "      <td>17.527233</td>\n",
              "      <td>...</td>\n",
              "      <td>6.335702</td>\n",
              "      <td>20.700720</td>\n",
              "      <td>29.276700</td>\n",
              "      <td>5.626096</td>\n",
              "      <td>15.481800</td>\n",
              "      <td>26.24797</td>\n",
              "      <td>2.623062</td>\n",
              "      <td>0.400118</td>\n",
              "      <td>24.339509</td>\n",
              "      <td>45.498040</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-03</td>\n",
              "      <td>25.217936</td>\n",
              "      <td>27.691776</td>\n",
              "      <td>26.745528</td>\n",
              "      <td>24.618408</td>\n",
              "      <td>33.900899</td>\n",
              "      <td>35.589010</td>\n",
              "      <td>33.690254</td>\n",
              "      <td>38.341235</td>\n",
              "      <td>34.809658</td>\n",
              "      <td>29.657188</td>\n",
              "      <td>...</td>\n",
              "      <td>2.452619</td>\n",
              "      <td>57.572800</td>\n",
              "      <td>4.243239</td>\n",
              "      <td>1.428393</td>\n",
              "      <td>35.264520</td>\n",
              "      <td>67.52889</td>\n",
              "      <td>2.677509</td>\n",
              "      <td>0.316854</td>\n",
              "      <td>208.070305</td>\n",
              "      <td>15.228980</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-04</td>\n",
              "      <td>38.854236</td>\n",
              "      <td>36.755066</td>\n",
              "      <td>35.572574</td>\n",
              "      <td>34.110252</td>\n",
              "      <td>48.683100</td>\n",
              "      <td>52.354600</td>\n",
              "      <td>49.098145</td>\n",
              "      <td>44.835977</td>\n",
              "      <td>43.202534</td>\n",
              "      <td>41.142932</td>\n",
              "      <td>...</td>\n",
              "      <td>5.044891</td>\n",
              "      <td>8.853860</td>\n",
              "      <td>7.402682</td>\n",
              "      <td>2.748726</td>\n",
              "      <td>9.494940</td>\n",
              "      <td>52.39232</td>\n",
              "      <td>4.041953</td>\n",
              "      <td>0.718312</td>\n",
              "      <td>55.862532</td>\n",
              "      <td>41.004660</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 7104 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                   X0         X1         X2         X3         X4         X5  \\\n",
              "2019-09-30  10.384020  12.222376   5.971256  11.441598  10.245286  14.770390   \n",
              "2019-10-01   2.303247   1.167996   3.119260   1.364694   2.355424   3.042923   \n",
              "2019-10-02  18.529622  16.354433  18.980730  14.328082  18.858808  20.995500   \n",
              "2019-10-03  25.217936  27.691776  26.745528  24.618408  33.900899  35.589010   \n",
              "2019-10-04  38.854236  36.755066  35.572574  34.110252  48.683100  52.354600   \n",
              "\n",
              "                   X6         X7         X8         X9  ...     X7094  \\\n",
              "2019-09-30  14.752220  16.288942  15.054422   9.530121  ...  3.265403   \n",
              "2019-10-01   3.324206   9.170274   2.758917   1.507425  ...  6.049601   \n",
              "2019-10-02  22.920832  18.492727  28.264308  17.527233  ...  6.335702   \n",
              "2019-10-03  33.690254  38.341235  34.809658  29.657188  ...  2.452619   \n",
              "2019-10-04  49.098145  44.835977  43.202534  41.142932  ...  5.044891   \n",
              "\n",
              "                X7095      X7096      X7097      X7098      X7099      X7100  \\\n",
              "2019-09-30   4.607365   5.798550  20.407620   2.930961  108.79449   2.566029   \n",
              "2019-10-01  10.028864  19.171390  36.775890  10.900753   19.79402  13.954580   \n",
              "2019-10-02  20.700720  29.276700   5.626096  15.481800   26.24797   2.623062   \n",
              "2019-10-03  57.572800   4.243239   1.428393  35.264520   67.52889   2.677509   \n",
              "2019-10-04   8.853860   7.402682   2.748726   9.494940   52.39232   4.041953   \n",
              "\n",
              "               X7101       X7102      X7103  \n",
              "2019-09-30  4.219114   -6.518932   5.080588  \n",
              "2019-10-01  0.923192    1.547703  18.794590  \n",
              "2019-10-02  0.400118   24.339509  45.498040  \n",
              "2019-10-03  0.316854  208.070305  15.228980  \n",
              "2019-10-04  0.718312   55.862532  41.004660  \n",
              "\n",
              "[5 rows x 7104 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5uLsE_KMbpZD"
      },
      "source": [
        "new_prediction.to_csv('new_corr_pred_end.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SPA2suQibpZE"
      },
      "source": [
        "## Prediction 完成"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yVzuKpfjbpZE"
      },
      "source": [
        "## 將 Observation 對齊 Prediction 日期"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wV3NYlqmbpZI",
        "outputId": "ba4c1e27-d8f0-4561-8247-71bcb3d42433"
      },
      "source": [
        "new_prediction.index = (new_prediction.index).astype(str)\n",
        "observation_z = pd.concat([observation_z, new_prediction], axis=1, join_axes=[new_prediction.index]).iloc[:, :time_step*station_num]\n",
        "observation_z.tail()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "C:\\Users\\User\\Anaconda3\\envs\\tf2_rpy2\\lib\\site-packages\\ipykernel_launcher.py:2: FutureWarning: The join_axes-keyword is deprecated. Use .reindex or .reindex_like on the result to achieve the same functionality.\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>...</th>\n",
              "      <th>7094</th>\n",
              "      <th>7095</th>\n",
              "      <th>7096</th>\n",
              "      <th>7097</th>\n",
              "      <th>7098</th>\n",
              "      <th>7099</th>\n",
              "      <th>7100</th>\n",
              "      <th>7101</th>\n",
              "      <th>7102</th>\n",
              "      <th>7103</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>2019-10-28</td>\n",
              "      <td>8.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>11.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>14.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>...</td>\n",
              "      <td>25.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>21.0</td>\n",
              "      <td>32.0</td>\n",
              "      <td>20.0</td>\n",
              "      <td>37.0</td>\n",
              "      <td>43.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>19.0</td>\n",
              "      <td>27.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-29</td>\n",
              "      <td>6.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-30</td>\n",
              "      <td>19.0</td>\n",
              "      <td>19.0</td>\n",
              "      <td>24.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>18.0</td>\n",
              "      <td>7.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>21.0</td>\n",
              "      <td>18.0</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-31</td>\n",
              "      <td>27.0</td>\n",
              "      <td>26.0</td>\n",
              "      <td>40.0</td>\n",
              "      <td>24.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>32.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>32.0</td>\n",
              "      <td>34.0</td>\n",
              "      <td>27.0</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-11-01</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 7104 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "            0     1     2     3     4     5     6     7     8     9     ...  \\\n",
              "2019-10-28   8.0   9.0   8.0   5.0  11.0  10.0  10.0  14.0  15.0  12.0  ...   \n",
              "2019-10-29   6.0   6.0   3.0   3.0   4.0   2.0   5.0   8.0   5.0   6.0  ...   \n",
              "2019-10-30  19.0  19.0  24.0  15.0  18.0   7.0  12.0  12.0  21.0  18.0  ...   \n",
              "2019-10-31  27.0  26.0  40.0  24.0  29.0  32.0  29.0  32.0  34.0  27.0  ...   \n",
              "2019-11-01   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
              "\n",
              "            7094  7095  7096  7097  7098  7099  7100  7101  7102  7103  \n",
              "2019-10-28  25.0  29.0  21.0  32.0  20.0  37.0  43.0   4.0  19.0  27.0  \n",
              "2019-10-29   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
              "2019-10-30   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
              "2019-10-31   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
              "2019-11-01   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
              "\n",
              "[5 rows x 7104 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 46
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tkwojtItbpZJ"
      },
      "source": [
        "observation_z.to_csv('corr_observation_z.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RBMP01cDzgWZ",
        "outputId": "bd9b243e-1eb4-48e4-f225-67e8039a3757"
      },
      "source": [
        "new_prediction.index = (new_prediction.index).astype(str)\n",
        "observation_y = pd.concat([observation_y, new_prediction], axis=1, join_axes=[new_prediction.index]).iloc[:, :time_step*station_num]\n",
        "observation_y.tail()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "C:\\Users\\User\\Anaconda3\\envs\\tf2_rpy2\\lib\\site-packages\\ipykernel_launcher.py:2: FutureWarning: The join_axes-keyword is deprecated. Use .reindex or .reindex_like on the result to achieve the same functionality.\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>...</th>\n",
              "      <th>7094</th>\n",
              "      <th>7095</th>\n",
              "      <th>7096</th>\n",
              "      <th>7097</th>\n",
              "      <th>7098</th>\n",
              "      <th>7099</th>\n",
              "      <th>7100</th>\n",
              "      <th>7101</th>\n",
              "      <th>7102</th>\n",
              "      <th>7103</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>2019-10-28</td>\n",
              "      <td>8.0</td>\n",
              "      <td>9.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>11.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>10.0</td>\n",
              "      <td>14.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>...</td>\n",
              "      <td>25.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>21.0</td>\n",
              "      <td>32.0</td>\n",
              "      <td>20.0</td>\n",
              "      <td>37.0</td>\n",
              "      <td>43.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>19.0</td>\n",
              "      <td>27.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-29</td>\n",
              "      <td>6.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>3.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>2.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>8.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>6.0</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-30</td>\n",
              "      <td>19.0</td>\n",
              "      <td>19.0</td>\n",
              "      <td>24.0</td>\n",
              "      <td>15.0</td>\n",
              "      <td>18.0</td>\n",
              "      <td>7.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>12.0</td>\n",
              "      <td>21.0</td>\n",
              "      <td>18.0</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-10-31</td>\n",
              "      <td>27.0</td>\n",
              "      <td>26.0</td>\n",
              "      <td>40.0</td>\n",
              "      <td>24.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>32.0</td>\n",
              "      <td>29.0</td>\n",
              "      <td>32.0</td>\n",
              "      <td>34.0</td>\n",
              "      <td>27.0</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2019-11-01</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 7104 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "            0     1     2     3     4     5     6     7     8     9     ...  \\\n",
              "2019-10-28   8.0   9.0   8.0   5.0  11.0  10.0  10.0  14.0  15.0  12.0  ...   \n",
              "2019-10-29   6.0   6.0   3.0   3.0   4.0   2.0   5.0   8.0   5.0   6.0  ...   \n",
              "2019-10-30  19.0  19.0  24.0  15.0  18.0   7.0  12.0  12.0  21.0  18.0  ...   \n",
              "2019-10-31  27.0  26.0  40.0  24.0  29.0  32.0  29.0  32.0  34.0  27.0  ...   \n",
              "2019-11-01   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
              "\n",
              "            7094  7095  7096  7097  7098  7099  7100  7101  7102  7103  \n",
              "2019-10-28  25.0  29.0  21.0  32.0  20.0  37.0  43.0   4.0  19.0  27.0  \n",
              "2019-10-29   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
              "2019-10-30   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
              "2019-10-31   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
              "2019-11-01   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
              "\n",
              "[5 rows x 7104 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xyS5x4zJzgWc"
      },
      "source": [
        "observation_y.to_csv('corr_observation_y.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PsxQ2DsibpZK"
      },
      "source": [
        "## Observation 完成"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zkHnAGC8bpZK"
      },
      "source": [
        "# ----------------------------------------------------------------------------------------"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J2QnP_l1bpZL"
      },
      "source": [
        "## Model 4"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ojyXV0IQbpZL"
      },
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow.keras import layers\n",
        "from sklearn.model_selection import train_test_split, GridSearchCV\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "so8BFhCwzgWh"
      },
      "source": [
        "config = tf.compat.v1.ConfigProto()\n",
        "config.gpu_options.allow_growth = True\n",
        "sess = tf.compat.v1.Session(config=config)\n",
        "config = tf.compat.v1.ConfigProto()\n",
        "config.gpu_options.per_process_gpu_memory_fraction = 0.95\n",
        "tf.compat.v1.Session(config=config)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ke9l4wMgzgWj"
      },
      "source": [
        "new_prediction = pd.read_csv('new_corr_pred_end_96.csv')\n",
        "new_prediction.index = new_prediction['Unnamed: 0']\n",
        "new_prediction = new_prediction.drop('Unnamed: 0',axis = 1)\n",
        "\n",
        "observation_y = pd.read_csv('corr_observation_y_96.csv')\n",
        "observation_y.index = observation_y['Unnamed: 0']\n",
        "observation_y = observation_y.drop('Unnamed: 0',axis = 1)\n",
        "\n",
        "observation_z = pd.read_csv('corr_observation_z_96.csv')\n",
        "observation_z.index = observation_z['Unnamed: 0']\n",
        "observation_z = observation_z.drop('Unnamed: 0',axis = 1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "06fm49dPbpZM"
      },
      "source": [
        "XData = np.array(new_prediction)[1:,:].astype('float32')\n",
        "ZData = np.array(observation_z)[:-1,0:station_num*24].astype('float32')  #往前看24小時\n",
        "YData = np.array(observation_y)[1:,:].astype('float32')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vsjwka53ni8M"
      },
      "source": [
        "## 比較校正後的結果"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MpVvFvsrbpZf",
        "outputId": "a4c11a55-fdae-49eb-df80-24b4257ae98c"
      },
      "source": [
        "new_model = tf.keras.models.load_model('model4_time_noactivation_mse.h5',compile=False)\n",
        "y_hat = new_model([ ZData], training = False).numpy()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:No training configuration found in save file: the model was *not* compiled. Compile it manually.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "m--257l7bpZg"
      },
      "source": [
        "## 將資料整理回原格式"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kIX3q5Jani8T",
        "outputId": "b6b30982-e690-46ee-fb52-86ea19efbc8c"
      },
      "source": [
        "stations = np.unique(df_prediction['stationID'])\n",
        "correction = list(y_hat.reshape(-1,1).flatten())\n",
        "Pred = list(XData.flatten())\n",
        "expdate = (pd.to_datetime(new_prediction.index)[1:] - timedelta(days=1)).astype(str).repeat(time_step*station_num)\n",
        "stationID = list(stations) * time_step *len(XData)\n",
        "tau = list((np.arange(time_step) + 1).repeat(len(stations))) * len(XData)\n",
        "hour = np.array(tau) % 24\n",
        "prediction = { \n",
        "             'expdate':expdate,\n",
        "             'stationID':stationID,\n",
        "             'tau':tau,\n",
        "             'hour':hour,\n",
        "             \"PMf_ori\": Pred,\n",
        "             \"PMf_corr\": correction}\n",
        "prediction_df = pd.DataFrame(prediction)\n",
        "prediction_df['day'] = (pd.to_datetime(prediction_df['expdate']) + \n",
        "                        pd.TimedeltaIndex(23 + prediction_df['tau'], unit='h')).dt.day\n",
        "prediction_df['month'] = (pd.to_datetime(prediction_df['expdate']) + \n",
        "                        pd.TimedeltaIndex(23 + prediction_df['tau'], unit='h')).dt.month\n",
        "prediction_df['year'] = (pd.to_datetime(prediction_df['expdate']) + \n",
        "                        pd.TimedeltaIndex(23 + prediction_df['tau'], unit='h')).dt.year\n",
        "prediction_df = prediction_df[['expdate','stationID','year','month','day','hour','tau','PMf_ori','PMf_corr']]\n",
        "prediction_df['observation'] = np.array(observation_y[1:]).flatten(0)\n",
        "prediction_df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "C:\\Users\\user\\Anaconda3\\envs\\tf2gpu\\lib\\site-packages\\ipykernel_launcher.py:23: DeprecationWarning: Non-string object detected for the array ordering. Please pass in 'C', 'F', 'A', or 'K' instead\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>expdate</th>\n",
              "      <th>stationID</th>\n",
              "      <th>year</th>\n",
              "      <th>month</th>\n",
              "      <th>day</th>\n",
              "      <th>hour</th>\n",
              "      <th>tau</th>\n",
              "      <th>PMf_ori</th>\n",
              "      <th>PMf_corr</th>\n",
              "      <th>observation</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2019-09-30</td>\n",
              "      <td>EPA001</td>\n",
              "      <td>2019</td>\n",
              "      <td>10</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2.303247</td>\n",
              "      <td>9.142049</td>\n",
              "      <td>6.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2019-09-30</td>\n",
              "      <td>EPA002</td>\n",
              "      <td>2019</td>\n",
              "      <td>10</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1.167996</td>\n",
              "      <td>11.435400</td>\n",
              "      <td>5.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2019-09-30</td>\n",
              "      <td>EPA003</td>\n",
              "      <td>2019</td>\n",
              "      <td>10</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>3.119260</td>\n",
              "      <td>9.922529</td>\n",
              "      <td>8.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2019-09-30</td>\n",
              "      <td>EPA004</td>\n",
              "      <td>2019</td>\n",
              "      <td>10</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1.364694</td>\n",
              "      <td>8.797889</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2019-09-30</td>\n",
              "      <td>EPA005</td>\n",
              "      <td>2019</td>\n",
              "      <td>10</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2.355424</td>\n",
              "      <td>13.033556</td>\n",
              "      <td>5.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      expdate stationID  year  month  day  hour  tau   PMf_ori   PMf_corr  \\\n",
              "0  2019-09-30    EPA001  2019     10    1     1    1  2.303247   9.142049   \n",
              "1  2019-09-30    EPA002  2019     10    1     1    1  1.167996  11.435400   \n",
              "2  2019-09-30    EPA003  2019     10    1     1    1  3.119260   9.922529   \n",
              "3  2019-09-30    EPA004  2019     10    1     1    1  1.364694   8.797889   \n",
              "4  2019-09-30    EPA005  2019     10    1     1    1  2.355424  13.033556   \n",
              "\n",
              "   observation  \n",
              "0          6.0  \n",
              "1          5.0  \n",
              "2          8.0  \n",
              "3          1.0  \n",
              "4          5.0  "
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l-k9A6MfzgWr"
      },
      "source": [
        "def error_print(df):\n",
        "    def error(Y_hat, real):\n",
        "        MNE = np.nanmean(abs((Y_hat[real!=0] - real[real!=0])/(real[real!=0]))) \n",
        "        MNB = np.nanmean((Y_hat[real!=0] - real[real!=0])/(real[real!=0]))\n",
        "        return MNE, MNB\n",
        "    df = df.dropna(subset=[\"observation\"])  \n",
        "                      \n",
        "    MNE_corr, MNB_corr = error(df['PMf_corr'], df['observation'])  \n",
        "    print('regression Mean Error = ', np.nanmean(abs(df['PMf_corr']-df['observation'])))\n",
        "    print('regression Mean Normalized Error = ', MNE_corr)\n",
        "    print('regression Mean Normalized Bias = ', MNB_corr)\n",
        "    print('regression correlation coefficient = ', df.PMf_corr.corr(df.observation))\n",
        "    print(' ')                  \n",
        "    MNE_ori, MNB_ori = error(df['PMf_ori'], df['observation'])\n",
        "    print('original Mean  Absolute Error = ', np.nanmean(abs(df['PMf_ori']-df['observation'])))\n",
        "    print('original Mean Normalized Error = ', MNE_ori)\n",
        "    print('original Mean Normalized Bias = ', MNB_ori)\n",
        "    print('original correlation coefficient = ', df.PMf_ori.corr(df.observation))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lcdtAIYyzgWs",
        "outputId": "818cb448-8d0f-4cc3-e929-c8c149a5af14"
      },
      "source": [
        "error_print(prediction_df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "regression Mean Error =  8.351368074574367\n",
            "regression Mean Normalized Error =  0.36967927001183754\n",
            "regression Mean Normalized Bias =  0.08535100917355362\n",
            "regression correlation coefficient =  0.4750865989424881\n",
            " \n",
            "original Mean  Absolute Error =  18.775480663105128\n",
            "original Mean Normalized Error =  0.7581320672885375\n",
            "original Mean Normalized Bias =  -0.3603805984944449\n",
            "original correlation coefficient =  0.15693735320468338\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mpdfeSLPzgWu"
      },
      "source": [
        "prediction_df.to_csv('after_corr_noactivate_mse_time.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZXgDajvhni8V",
        "outputId": "ca106a3a-3ff2-4bf9-ceb7-5a161a3d7988"
      },
      "source": [
        "\n",
        "prediction_df.to_csv(sys.argv[5] + '/' + sys.argv[6])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "IndexError",
          "evalue": "list index out of range",
          "traceback": [
            "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[1;31mIndexError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[1;32m<ipython-input-57-df285bb8d2d4>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mprediction_df\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0margv\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m'/'\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0msys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0margv\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m6\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[1;31mIndexError\u001b[0m: list index out of range"
          ]
        }
      ]
    }
  ]
}