Newdam-Copy2 no graph.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data=pd.read_csv('~/complete-data-tll-2017-march-2018.csv')\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data=data.drop(data.index[[0,1,2]])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = data.iloc[:,0:2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=data.replace({'00:00': ''}, regex=True)\n",
    "data=data.replace({'/': '-'}, regex=True)\n",
    "#data\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data.columns = ['Time','Discharge']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data.to_csv(\"~/XNEWdamkc1.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "data2=pd.read_csv('~/XNEWdamkc1.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data2\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "data2 = data2.drop(data2.columns[[0]],axis = 1)\n",
    "#data2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data3=data2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "traindata=data3.iloc[0:]\n",
    "#traindata\n",
    "#data3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#testdata=data3.iloc[22639:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "#testdata.head()\n",
    "#traindata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "traindata.to_csv(\"~/XNEWtrainDam.csv\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#testdata.to_csv(\"~/testDam.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dateparse = lambda dates: pd.datetime.strptime(dates, '%d-%m-%Y ')\n",
    "traindata = pd.read_csv('~/XNEWtrainDam.csv', parse_dates=['Time'], index_col='Time',date_parser=dateparse)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "#traindata\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "traindata= traindata.drop(traindata.columns[[0]],axis = 1)\n",
    "#traindata\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "#traindata.index\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "#np.where(pd.isnull(traindata))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(traindata.describe())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pd.isnull(traindata).any()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "#traindata.isnull().values.any()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#testdata.to_csv(\"~/testDam2.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "newTrainData= traindata.interpolate(method='time')\n",
    "#newTrainData\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "#pd.isnull(traindata).any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "#np.where(pd.isnull(newTrainData))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#import matplotlib.pylab as plt\n",
    "#%matplotlib inline\n",
    "#from matplotlib.pylab import rcParams\n",
    "#rcParams['figure.figsize'] = 15, 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "#plt.plot(newTrainData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "#plt.plot(traindata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ax = newTrainData.plot()\n",
    "#traindata.plot(ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "newTrainData.to_csv(\"~/XtrainDam3.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "#from statsmodels.tsa.stattools import adfuller\n",
    "#def test_stationarity(timeseries):\n",
    "    \n",
    "    #Determing rolling statistics\n",
    " #   rolmean = pd.rolling_mean(timeseries, window=12)\n",
    "  #  rolstd = pd.rolling_std(timeseries, window=12)\n",
    "\n",
    "    #Plot rolling statistics:\n",
    "    #orig = plt.plot(timeseries, color='blue',label='Original')\n",
    "    #mean = plt.plot(rolmean, color='red', label='Rolling Mean')\n",
    "    #std = plt.plot(rolstd, color='black', label = 'Rolling Std')\n",
    "  #  plt.legend(loc='best')\n",
    "   # plt.title('Rolling Mean & Standard Deviation')\n",
    "   # plt.show(block=False)\n",
    "    \n",
    "    #Perform Dickey-Fuller test:\n",
    "    #print ('Results of Dickey-Fuller Test:')\n",
    "    #dftest = adfuller(timeseries, autolag='AIC')\n",
    "   # dfoutput = pd.Series(dftest[0:4], index=['Test Statistic','p-value','#Lags Used','Number of Observations Used'])\n",
    "  #  for key,value in dftest[4].items():\n",
    " #       dfoutput['Critical Value (%s)'%key] = value\n",
    "#    print (dfoutput)\n",
    "\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test_stationarity(newTrainData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "trainset=newTrainData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "#trainset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "trainset = trainset.iloc[:,0].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "#trainset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#test_stationarity(trainset)\n",
    "#newTrainData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#        from statsmodels.tsa.seasonal import seasonal_decompose\n",
    " #       decomposition = seasonal_decompose(newTrainData)\n",
    "\n",
    "#        trend = decomposition.trend\n",
    " #       seasonal = decomposition.seasonal\n",
    " #       residual = decomposition.resid\n",
    "\n",
    "  #      plt.subplot(411)\n",
    "   #     plt.plot(newTrainData, label='Original')\n",
    "    #    plt.legend(loc='best')\n",
    "     #   plt.subplot(412)\n",
    "      #  plt.plot(trend, label='Trend')\n",
    "       # plt.legend(loc='best')\n",
    "       # plt.subplot(413)\n",
    "      #  plt.plot(seasonal,label='Seasonality')\n",
    "     #   plt.legend(loc='best')\n",
    "    #    plt.subplot(414)\n",
    "  #      plt.plot(residual, label='Residuals')\n",
    "  #      plt.legend(loc='best')\n",
    "#        plt.tight_layout()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ts_log_decompose = residual\n",
    "#ts_log_decompose.dropna(inplace=True)\n",
    "#test_stationarity(ts_log_decompose)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#mkc=ts_log_decompose.iloc[:,0].values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# mkc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test_stationarity(mkc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "#from statsmodels.tsa.stattools import acf, pacf\n",
    "#lag_acf = acf(newTrainData, nlags=20)\n",
    "#lag_pacf = pacf(newTrainData, nlags=20, method='ols')\n",
    "#Plot ACF: \n",
    "#plt.subplot(121) \n",
    "#plt.plot(lag_acf)\n",
    "#plt.axhline(y=0,linestyle='--',color='gray')\n",
    "#plt.axhline(y=-1.96/np.sqrt(len(newTrainData)),linestyle='--',color='gray')\n",
    "#plt.axhline(y=1.96/np.sqrt(len(newTrainData)),linestyle='--',color='gray')\n",
    "#plt.title('Autocorrelation Function')\n",
    "#Plot PACF:\n",
    "#plt.subplot(122)\n",
    "#plt.plot(lag_pacf)\n",
    "#plt.axhline(y=0,linestyle='--',color='gray')\n",
    "#plt.axhline(y=-1.96/np.sqrt(len(newTrainData)),linestyle='--',color='gray')\n",
    "#plt.axhline(y=1.96/np.sqrt(len(newTrainData)),linestyle='--',color='gray')\n",
    "#plt.title('Partial Autocorrelation Function')\n",
    "#plt.tight_layout()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "#newTrainData.head(-15)\n",
    "#trend\n",
    "#residual"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "from statsmodels.tsa.arima_model import ARIMA\n",
    "model = ARIMA(newTrainData, order=(2,0,0) ) \n",
    "results_ARIMA = model.fit()  \n",
    "\n",
    "#plt.plot(newTrainData)\n",
    "#plt.plot(results_ARIMA.fittedvalues, color='red')\n",
    "#plt.title('RSS: %.4f'% sum((results_ARIMA.fittedvalues-ts)**2))\n",
    "#plt.xlabel('DATE', fontsize = 20)\n",
    "#plt.ylabel('Water Discharge', fontsize = 20)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2018-03-28    1000.698650\n",
       "2018-03-29    1010.537078\n",
       "2018-03-30    1008.310579\n",
       "2018-03-31    1018.169368\n",
       "2018-04-01    1005.805545\n",
       "2018-04-02    1306.922927\n",
       "2018-04-03    1553.782166\n",
       "2018-04-04    1746.199316\n",
       "2018-04-05    1893.998920\n",
       "2018-04-06    2007.024096\n",
       "2018-04-07    2093.339054\n",
       "2018-04-08    2159.228372\n",
       "2018-04-09    2209.519096\n",
       "2018-04-10    2247.902501\n",
       "2018-04-11    2277.197519\n",
       "Freq: D, dtype: float64"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "start_index = '2018-03-28'\n",
    "end_index = '2018-04-11'\n",
    "forecast = results_ARIMA.predict(start=start_index, end=end_index)\n",
    "#plt.bar(forecast.index,forecast,width=10)\n",
    "#plt.title('NEXT YEAR PREDICTION',fontsize = 20)\n",
    "#plt.xlabel('DATE', fontsize = 20)\n",
    "#plt.ylabel('Water Discharge', fontsize = 20)\n",
    "#forecast\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "#forecast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "forecast.to_csv(\"XMKB.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}