WRN_ResNeXt.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import h5py\n",
    "import os\n",
    "import numpy as np\n",
    "\n",
    "def unpickle(file):\n",
    "    import cPickle\n",
    "    with open(file, 'rb') as fo:\n",
    "        dict = cPickle.load(fo)\n",
    "    return dict\n",
    "\n",
    "file = h5py.File('processed_data.h5','r+') \n",
    "\n",
    "#Retrieves all the preprocessed training and validation\\testing data from a file\n",
    "\n",
    "X_train = file['X_train'][...]\n",
    "Y_train = file['Y_train'][...]\n",
    "X_val = file['X_val'][...]\n",
    "Y_val = file['Y_val'][...]\n",
    "X_test = file['X_test'][...]\n",
    "Y_test = file['Y_test'][...]\n",
    "\n",
    "# Unpickles and retrieves class names and other meta informations of the database\n",
    "classes = unpickle('cifar-10-batches-py/batches.meta') #keyword for label = label_names\n",
    "\n",
    "print(\"Training sample shapes (input and output): \"+str(X_train.shape)+\" \"+str(Y_train.shape))\n",
    "print(\"Validation sample shapes (input and output): \"+str(X_val.shape)+\" \"+str(Y_val.shape))\n",
    "print(\"Testing sample shapes (input and output): \"+str(X_test.shape)+\" \"+str(Y_test.shape))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Creates nested list. The outer list will list all the classess (0-9). And each of the classes represent the inner list which list all\n",
    "#training data that belongs to that class. I used list because it is easy to keep on adding dynamically. Ndarrays may have needed \n",
    "#a predifined shape\n",
    "\n",
    "classes_num = len(classes['label_names']) #classes_num = no. of classes\n",
    "\n",
    "# Here, I am creating a special variable X_train_F which is basically a nested list.\n",
    "# The outermost list of X_train_F will be a list of all the class values (0-9 where each value correspond to a class name)\n",
    "# Each elements (class values) of the outermost list is actually also a list; a list of all the example data belonging\n",
    "# to the particular class which corresponds to class value under which the data is listed. \n",
    "\n",
    "X_train_F = []\n",
    "\n",
    "for i in xrange(0,classes_num):\n",
    "    X_train_F.append([])\n",
    "\n",
    "\n",
    "for i in xrange(0,len(X_train)):\n",
    "    l = np.argmax(Y_train[i]) #l for label (in this case it's basically the index of class value elemenmts)  \n",
    "    #(Y_train is one hot encoded. Argmax returns the index for maximum value which should be 1 and\n",
    "    # that index should indicate the value)\n",
    "    X_train_F[l].append(X_train[i])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "from scipy.misc import toimage\n",
    "from scipy.misc import imresize\n",
    "%matplotlib inline\n",
    "\n",
    "#function for showing pictures in grid along with labels\n",
    "\n",
    "def picgrid(X_train,Y_train,gray=0):\n",
    "    plt.figure(figsize=(7,7))\n",
    "    ax=[]\n",
    "    for i in xrange(0,25):\n",
    "        img = toimage(X_train[i])\n",
    "        ax.append(plt.subplot(5,5,i+1))\n",
    "        ax[i].set_title( classes['label_names'][np.argmax(Y_train[i])],y=-0.3)\n",
    "        ax[i].set_axis_off()\n",
    "        if gray==0:\n",
    "            plt.imshow(img)\n",
    "        else:\n",
    "            plt.imshow(img,cmap='gray')\n",
    "    plt.subplots_adjust(hspace=0.3)\n",
    "    plt.axis('off')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "\n",
    "smoothing_factor = 0.1 #for label smoothing\n",
    "\n",
    "def create_batches(batch_size,classes_num):\n",
    "   \n",
    "    s = int(batch_size/classes_num) #s denotes samples taken from each class to create the batch.\n",
    "    no_of_batches = int(len(X_train)/batch_size)\n",
    "    \n",
    "    shuffled_indices_per_class =[]\n",
    "    for i in xrange(0,classes_num):\n",
    "        temp = np.arange(len(X_train_F[i]))\n",
    "        np.random.shuffle(temp)\n",
    "        shuffled_indices_per_class.append(temp)\n",
    "        \n",
    "    batches_X = []\n",
    "    batches_Y = []\n",
    "        \n",
    "    for i in xrange(no_of_batches):\n",
    "        \n",
    "        shuffled_class_indices = np.arange(classes_num)\n",
    "        np.random.shuffle(shuffled_class_indices)\n",
    "        \n",
    "        batch_Y = np.zeros((batch_size,classes_num),np.float32)\n",
    "        batch_X = np.zeros((batch_size,32,32,3),np.float32)\n",
    "        \n",
    "        for index in xrange(0,classes_num):\n",
    "            class_index = shuffled_class_indices[index]\n",
    "            for j in xrange(0,s):\n",
    "                batch_X[(index*s)+j] = X_train_F[class_index][shuffled_indices_per_class[class_index][i*s+j]] # Assign the s chosen random samples to the training batch\n",
    "                batch_Y[(index*s)+j][class_index] = 1\n",
    "                batch_Y[(index*s)+j] = (1-smoothing_factor)*batch_Y[(index*s)+j] + smoothing_factor/classes_num\n",
    "        \n",
    "        rs = batch_size - s*classes_num #rs denotes no. of random samples from random classes to take\n",
    "                                        #in order to fill the batch if batch isn't divisble by classes_num\n",
    "        #fill the rest of the batch with random data\n",
    "        rand = random.sample(np.arange(len(X_train)),rs)\n",
    "        j=0\n",
    "        for k in xrange(s*classes_num,batch_size):\n",
    "            batch_X[k] = X_train[int(rand[j])]\n",
    "            batch_Y[k] = Y_train[int(rand[j])]\n",
    "            batch_Y[k] = (1-smoothing_factor)*batch_Y[k] + smoothing_factor/classes_num\n",
    "            j+=1\n",
    "\n",
    "        batches_X.append(batch_X)\n",
    "        batches_Y.append(batch_Y)\n",
    "    \n",
    "    return batches_X,batches_Y\n",
    "\n",
    "batches_X,batches_Y = create_batches(64,classes_num) # A demo of the function at work\n",
    "\n",
    "# Since each batch will have almost equal no. of cases from each class, no batch should be biased towards some particular classes\n",
    "\n",
    "sample = random.randint(0,len(batches_X))\n",
    "print \"Sample arranged images in a batch: \"\n",
    "picgrid(batches_X[sample],batches_Y[sample])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_crop(img):\n",
    "    #result = np.zeros_like((img))\n",
    "    c = np.random.randint(0,5)\n",
    "    if c==0:\n",
    "        crop = img[4:32,0:-4]\n",
    "    elif c==1:\n",
    "        crop = img[0:-4,0:-4]\n",
    "    elif c==2:\n",
    "        crop = img[2:-2,2:-2]\n",
    "    elif c==3:\n",
    "        crop = img[4:32,4:32]\n",
    "    elif c==4:\n",
    "        crop = img[0:-4,4:32]\n",
    "    \n",
    "    #translating cropped position\n",
    "    #over the original image\n",
    "    c = np.random.randint(0,5)\n",
    "    if c==0:\n",
    "        img[4:32,0:-4] = crop[:]\n",
    "    elif c==1:\n",
    "        img[0:-4,0:-4] = crop[:]\n",
    "    elif c==2:\n",
    "        img[2:-2,2:-2] = crop[:]\n",
    "    elif c==3:\n",
    "        img[4:32,4:32] = crop[:]\n",
    "    elif c==4:\n",
    "        img[0:-4,4:32] = crop[:]\n",
    "        \n",
    "    return img\n",
    "\n",
    "def augment_batch(batch_X): #will be used to modify images realtime during training (real time data augmentation)\n",
    "    \n",
    "    aug_batch_X = np.zeros((len(batch_X),32,32,3))\n",
    "   \n",
    "    for i in xrange(0,len(batch_X)):\n",
    "        \n",
    "        hf = np.random.randint(0,2)\n",
    "        \n",
    "        if hf == 1: #hf denotes horizontal flip. 50-50 random chance to apply horizontal flip on images,\n",
    "            batch_X[i] = np.fliplr(batch_X[i])\n",
    "       \n",
    "        # Remove the below cropping to apply random crops. But before that it's better to implement something like mirror padding\n",
    "        # or any form of padding to increase the dimensions beforehand.\n",
    "        \n",
    "        c = np.random.randint(0,3)\n",
    "        if c==1:\n",
    "           #one in a three chance for cropping\n",
    "           #randomly crop 28x28 portions and translate it.\n",
    "            aug_batch_X[i] = random_crop(batch_X[i])\n",
    "        else:\n",
    "            aug_batch_X[i] = batch_X[i]\n",
    "    \n",
    "    return aug_batch_X\n",
    "    \n",
    "aug_batches_X=[]\n",
    "for batch in batches_X:\n",
    "    aug_batch_X = augment_batch(batch)\n",
    "    aug_batches_X.append(aug_batch_X)\n",
    "\n",
    "print \"Sample batch training images after augmentation:\"\n",
    "picgrid(aug_batches_X[sample],batches_Y[sample])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def shuffle_batch(batch_X,batch_Y):\n",
    "    shuffle = random.sample(np.arange(0,len(batch_X),1,'int'),len(batch_X))\n",
    "    shuffled_batch_X = []\n",
    "    shuffled_batch_Y = []\n",
    "    \n",
    "    for i in xrange(0,len(batch_X)):\n",
    "        shuffled_batch_X.append(batch_X[int(shuffle[i])])\n",
    "        shuffled_batch_Y.append(batch_Y[int(shuffle[i])])\n",
    "    \n",
    "    shuffled_batch_X = np.array(shuffled_batch_X)\n",
    "    shuffled_batch_Y = np.array(shuffled_batch_Y)\n",
    "\n",
    "    return shuffled_batch_X,shuffled_batch_Y\n",
    "\n",
    "\n",
    "\n",
    "s_batches_X=[]\n",
    "s_batches_Y=[]\n",
    "for i in xrange(len(aug_batches_X)):\n",
    "    s_batch_X,s_batch_Y = shuffle_batch(aug_batches_X[i],batches_Y[i])\n",
    "    s_batches_X.append(s_batch_X)\n",
    "    s_batches_Y.append(s_batch_Y)\n",
    "\n",
    "print \"Sample batch training images after shuffling\"\n",
    "picgrid(s_batches_X[sample],s_batches_Y[sample])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def batch(batch_size): #one shortcut function to execute all necessary functions to create a training batch\n",
    "    batches_X,batches_Y = create_batches(batch_size,classes_num)\n",
    "    \n",
    "    aug_batches_X=[]\n",
    "    for batch in batches_X:\n",
    "        aug_batch_X = augment_batch(batch)\n",
    "        aug_batches_X.append(aug_batch_X)\n",
    "        \n",
    "    s_batches_X=[]\n",
    "    s_batches_Y=[]\n",
    "    \n",
    "    for i in xrange(len(aug_batches_X)):\n",
    "        s_batch_X,s_batch_Y = shuffle_batch(aug_batches_X[i],batches_Y[i])\n",
    "        s_batches_X.append(s_batch_X)\n",
    "        s_batches_Y.append(s_batch_Y)\n",
    "    \n",
    "    return s_batches_X,s_batches_Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "#Hyper Parameters!\n",
    "\n",
    "learning_rate = 0.01\n",
    "init_lr = learning_rate\n",
    "batch_size = 64\n",
    "epochs = 500\n",
    "layers = 16\n",
    "beta = 0.0001 #l2 regularization scale\n",
    "ensemble = 2 #total no. of classifier models for ensembling\n",
    "\n",
    "K = 8 #(deepening factor)\n",
    "cardinality = 4*K \n",
    "\n",
    "n_classes = classes_num # another useless step that I made due to certain reasons. \n",
    "\n",
    "# tf Graph input\n",
    "\n",
    "x = tf.placeholder(tf.float32, [None, 32, 32, 3])\n",
    "y = tf.placeholder(tf.float32, [None,classes_num])\n",
    "phase = tf.placeholder(tf.bool)\n",
    "keep_prob = tf.placeholder(tf.float32) #dropout (keep probability)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Create some wrappers for simplicity\n",
    "def conv2d(x,shape,strides,scope):\n",
    "    # Conv2D wrapper\n",
    "    with tf.variable_scope(scope+\"regularize\",reuse=False):\n",
    "        W = tf.Variable(tf.truncated_normal(shape=shape,stddev=5e-2))\n",
    "    b = tf.Variable(tf.truncated_normal(shape=[shape[3]]))\n",
    "    x = tf.nn.conv2d(x, W, strides=[1, strides, strides, 1], padding='SAME')\n",
    "    x = tf.nn.bias_add(x, b)\n",
    "    return x\n",
    "\n",
    "def activate(x,phase):\n",
    "    #wrapper for performing batch normalization and elu activation\n",
    "    x = tf.contrib.layers.batch_norm(x, center=True, scale=True,variables_collections=[\"batch_norm_non_trainable_variables_collection\"],updates_collections=None, decay=0.9,is_training=phase,zero_debias_moving_mean=True, fused=True)\n",
    "    return tf.nn.elu(x)\n",
    "\n",
    "\n",
    "def wideres33block(X,N,K,iw,bw,s,dropout,phase,cardinality,scope):\n",
    "    \n",
    "    # Creates N no. of 3,3 type residual blocks with dropout that consitute the conv2/3/4 blocks\n",
    "    # with widening factor K and X as input. s is stride and bw is base width (no. of filters before multiplying with k)\n",
    "    # iw is input width.\n",
    "    # (see https://arxiv.org/abs/1605.07146 paper for details on the block)\n",
    "    # In this case, dropout = probability to keep the neuron enabled.\n",
    "    # phase = true when training, false otherwise.\n",
    "    \n",
    "    branch_filters = int((bw*K)/cardinality)\n",
    "    \n",
    "    conv33 = tf.constant(0,tf.float32)\n",
    "    \n",
    "    for i in xrange(cardinality):\n",
    "        \n",
    "        conv33_1 = conv2d(X,[3,3,iw,branch_filters],s,scope)\n",
    "        conv33_1 = activate(conv33_1,phase)\n",
    "        conv33_1 = tf.nn.dropout(conv33_1,dropout)\n",
    "    \n",
    "        conv33_2 = conv2d(conv33_1,[3,3,branch_filters,bw*K],1,scope)\n",
    "        \n",
    "        conv33 = tf.add(conv33_2,conv33)\n",
    "        \n",
    "    conv_skip= conv2d(X,[1,1,iw,bw*K],s,scope) #shortcut connection\n",
    "    caddtable = tf.add(conv33,conv_skip)\n",
    "    \n",
    "    #1st of the N blocks for conv2/3/4 block ends here. The rest of N-1 blocks will be implemented next with a loop.\n",
    "\n",
    "    for i in range(0,N-1):\n",
    "        \n",
    "        C = caddtable\n",
    "        Cactivated = activate(C,phase)\n",
    "        \n",
    "        conv33 = tf.constant(0,tf.float32)\n",
    "\n",
    "        for j in xrange(cardinality):\n",
    "            \n",
    "            conv33_1 = conv2d(Cactivated,[3,3,bw*K,branch_filters],1,scope)\n",
    "            conv33_1 = activate(conv33_1,phase)\n",
    "            conv33_1 = tf.nn.dropout(conv33_1,dropout)\n",
    "            \n",
    "            conv33_2 = conv2d(conv33_1,[3,3,branch_filters,bw*K],1,scope)\n",
    "            \n",
    "            conv33 = tf.add(conv33_2,conv33)\n",
    "            \n",
    "        caddtable = tf.add(conv33,C)\n",
    "    \n",
    "    return activate(caddtable,phase)\n",
    "\n",
    "\n",
    "    \n",
    "def WRN_ResNeXt(x,dropout, phase,layers,K,cardinality,scope): \n",
    "    \n",
    "    # 1 conv + 3 convblocks*(3 conv layers *1 group for each block + 2 conv layers*(N-1) groups for each block [total 1+N-1 = N groups]) = layers\n",
    "    # 3*2*(N-1) = layers - 1 - 3*3\n",
    "    # N = (layers -10)/6 + 1\n",
    "    # So N = (layers-4)/6\n",
    "\n",
    "    N = (layers-4)/6\n",
    "    \n",
    "    conv1 = conv2d(x,[3,3,3,16],1,scope)\n",
    "    conv1 = activate(conv1,phase)\n",
    "\n",
    "    conv2 = wideres33block(conv1,N,K,16,16,1,dropout,phase,cardinality,scope)\n",
    "    conv3 = wideres33block(conv2,N,K,16*K,32,2,dropout,phase,cardinality,scope)\n",
    "    conv4 = wideres33block(conv3,N,K,32*K,64,2,dropout,phase,cardinality,scope)\n",
    "\n",
    "    pooled = tf.nn.avg_pool(conv4,ksize=[1,8,8,1],strides=[1,1,1,1],padding='VALID')\n",
    "    \n",
    "    #Initialize weights and biases for fully connected layers\n",
    "    with tf.variable_scope(scope+\"regularize\",reuse=False):\n",
    "        wd1 = tf.Variable(tf.truncated_normal([1*1*64*K,32*K],stddev=5e-2))\n",
    "        wd2 = tf.Variable(tf.truncated_normal([32*K,16*K],stddev=5e-2))\n",
    "        wout = tf.Variable(tf.truncated_normal([16*K, n_classes]))\n",
    "    bd1 = tf.Variable(tf.constant(0.1,shape=[32*K]))\n",
    "    bd2 = tf.Variable(tf.constant(0.1,shape=[16*K]))\n",
    "    bout = tf.Variable(tf.constant(0.1,shape=[n_classes]))\n",
    "\n",
    "    # Fully connected layer\n",
    "    # Reshape pooling layer output to fit fully connected layer input\n",
    "    fc1 = tf.reshape(pooled, [-1, wd1.get_shape().as_list()[0]])   \n",
    "    fc1 = tf.add(tf.matmul(fc1, wd1), bd1)\n",
    "    fc1 = tf.nn.elu(fc1)\n",
    "    fc1 = tf.nn.dropout(fc1,keep_prob)\n",
    "  \n",
    "    fc2 = tf.add(tf.matmul(fc1, wd2), bd2)\n",
    "    fc2 = tf.nn.elu(fc2)\n",
    "\n",
    "    # Output, class prediction\n",
    "    out = tf.add(tf.matmul(fc2, wout), bout)\n",
    "    \n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Construct model\n",
    "\n",
    "ensembled_model = 0\n",
    "ensembled_cost = 0\n",
    "\n",
    "for i in xrange(ensemble):\n",
    "    model = WRN_ResNeXt(x,keep_prob,phase,layers=layers,K=K,cardinality=cardinality,scope=str(i))\n",
    "    #l2 regularization\n",
    "    weights = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope= str(i)+'regularize')\n",
    "    regularizer=0\n",
    "    for i in xrange(len(weights)):\n",
    "        regularizer += tf.nn.l2_loss(weights[i])\n",
    "    #cross entropy loss\n",
    "    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=model,labels=y) + beta*regularizer)\n",
    "    \n",
    "    float_ensemble = tf.cast(tf.constant(ensemble),tf.float32)\n",
    "    ensembled_model += model\n",
    "    ensembled_cost += cost\n",
    "\n",
    "ensembled_model += tf.div(ensembled_model,float_ensemble)\n",
    "ensembled_cost = tf.div(ensembled_cost,float_ensemble)\n",
    "\n",
    "global_step = tf.Variable(0, trainable=False)\n",
    "\n",
    "#optimizer \n",
    "optimizer = tf.train.MomentumOptimizer(learning_rate=learning_rate, \n",
    "                                       momentum = 0.9, \n",
    "                                       use_nesterov=True).minimize(ensembled_cost,global_step=global_step)\n",
    "\n",
    "# Evaluate model\n",
    "correct_pred = tf.equal(tf.argmax(ensembled_model,1),tf.argmax(y,1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "prediction = tf.nn.softmax(logits=ensembled_model)\n",
    "\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "with tf.Session() as sess: # Start Tensorflow Session\n",
    "    \n",
    "    saver = tf.train.Saver() # Prepares variable for saving the model\n",
    "    sess.run(init) #initialize all variables\n",
    "    step = 1   \n",
    "    loss_list=[]\n",
    "    acc_list=[]\n",
    "    val_loss_list=[]\n",
    "    val_acc_list=[]\n",
    "    best_val_acc=0\n",
    "    total_loss=0\n",
    "    total_acc=0\n",
    "    avg_loss=0\n",
    "    avg_acc=0\n",
    "    val_batch_size = batch_size\n",
    "    \n",
    "    \n",
    "    threshold = 0.5 #if training accuracy is 100-threshold or less, training will stop \n",
    "    \n",
    "    while step <= epochs:\n",
    "        \n",
    "        \n",
    "        # A little bit of Learning rate scheduling\n",
    "        if step == 120:\n",
    "            learning_rater = 0.004\n",
    "        elif step == 160:\n",
    "            learning_rate = 0.0008\n",
    "\n",
    "        \n",
    "        batches_X, batches_Y = batch(batch_size)\n",
    "        \n",
    "        for i in xrange(len(batches_X)):\n",
    "            # Run optimization operation (backpropagation)\n",
    "            _,loss,acc = sess.run([optimizer,cost,accuracy],\n",
    "                                   feed_dict={x: batches_X[i], y: batches_Y[i], \n",
    "                                   keep_prob: 0.7,\n",
    "                                   phase: True})\n",
    "            total_loss += loss\n",
    "            total_acc += acc\n",
    "            \n",
    "            if i%100 == 0:\n",
    "                print \"Iter \" + str((step-1)*len(batches_X)+i+1) + \", Minibatch Loss= \" + \\\n",
    "                  \"{:.3f}\".format(loss) + \", Minibatch Accuracy= \" + \\\n",
    "                  \"{:.3f}%\".format(acc*100)\n",
    "\n",
    "     \n",
    "                      \n",
    "        total_val_loss=0\n",
    "        total_val_acc=0\n",
    "        val_loss=0\n",
    "        val_acc=0\n",
    "        avg_val_loss=0\n",
    "        avg_val_acc=0\n",
    "            \n",
    "        i=0\n",
    "        count=0\n",
    "        while i<len(X_val):\n",
    "            \n",
    "            #print \"what's happening?\"\n",
    "            if i+val_batch_size<len(X_val):\n",
    "                val_loss, val_acc = sess.run([cost, accuracy], \n",
    "                                            feed_dict={x: X_val[i:i+val_batch_size],\n",
    "                                                       y: Y_val[i:i+val_batch_size],\n",
    "                                                       keep_prob: 1,\n",
    "                                                       phase: False})\n",
    "            else:\n",
    "                val_loss, val_acc = sess.run([cost, accuracy], \n",
    "                                            feed_dict={x: X_val[i:],\n",
    "                                                       y: Y_val[i:],\n",
    "                                                       keep_prob: 1,\n",
    "                                                       phase: False})\n",
    "                              \n",
    "            total_val_loss = total_val_loss + val_loss\n",
    "            total_val_acc = total_val_acc + val_acc\n",
    "            count+=1\n",
    "                \n",
    "            i+=val_batch_size\n",
    "  \n",
    "\n",
    " \n",
    "        avg_val_loss = total_val_loss/count # Average validation loss\n",
    "        avg_val_acc = total_val_acc/count # Average validation accuracy\n",
    "            \n",
    "             \n",
    "        val_loss_list.append(avg_val_loss) # Storing values in list for plotting later on.\n",
    "        val_acc_list.append(avg_val_acc) # Storing values in list for plotting later on.\n",
    "            \n",
    "        avg_loss = total_loss/len(batches_X) # Average mini-batch training loss\n",
    "        avg_acc = total_acc/len(batches_X)   # Average mini-batch training accuracy\n",
    "        loss_list.append(avg_loss) # Storing values in list for plotting later on.\n",
    "        acc_list.append(avg_acc) # Storing values in list for plotting later on.\n",
    "            \n",
    "        total_loss=0\n",
    "        total_acc=0\n",
    "\n",
    "        print \"\\nEpoch \" + str(step) + \", Validation Loss= \" + \\\n",
    "                \"{:.3f}\".format(avg_val_loss) + \", validation Accuracy= \" + \\\n",
    "                \"{:.3f}%\".format(avg_val_acc*100)+\"\"\n",
    "        print \"Epoch \" + str(step) + \", Average Training Loss= \" + \\\n",
    "                \"{:.3f}\".format(avg_loss) + \", Average Training Accuracy= \" + \\\n",
    "                \"{:.3f}%\".format(avg_acc*100)+\"\"\n",
    "                    \n",
    "        if avg_val_acc >= best_val_acc: # When better accuracy is received than previous best validation accuracy\n",
    "                \n",
    "            best_val_acc = avg_val_acc # update value of best validation accuracy received yet.\n",
    "            saver.save(sess, 'Model_Backup/model.ckpt') # save_model including model variables (weights, biases etc.)\n",
    "            print \"Checkpoint created!\"\n",
    "                \n",
    "                \n",
    "        print \"\"\n",
    "            \n",
    "        if (100-(avg_acc*100)) <= threshold:\n",
    "            print \"\\nConvergence Threshold Reached!\"\n",
    "            break\n",
    "              \n",
    "        step += 1\n",
    "        \n",
    "    print \"\\nOptimization Finished!\\n\"\n",
    "    \n",
    "    print \"Best Validation Accuracy: %.3f%%\"%((best_val_acc)*100)\n",
    "    \n",
    "    print 'Loading pre-trained weights for the model...'\n",
    "    saver = tf.train.Saver()\n",
    "    saver.restore(sess, 'Model_Backup/model.ckpt')\n",
    "    sess.run(tf.global_variables())\n",
    "    print '\\nRESTORATION COMPLETE\\n'\n",
    "    \n",
    "    print 'Testing Model Performance...'\n",
    "    test_batch_size = batch_size\n",
    "    total_test_loss=0\n",
    "    total_test_acc=0\n",
    "    test_loss=0\n",
    "    test_acc=0\n",
    "    avg_test_loss=0\n",
    "    avg_test_acc=0\n",
    "            \n",
    "    i=0\n",
    "    count=0\n",
    "    while i<len(X_test):\n",
    "        \n",
    "        if (i+test_batch_size)<len(X_test):\n",
    "            test_loss, test_acc = sess.run([cost, accuracy], \n",
    "                                         feed_dict={x: X_test[i:i+test_batch_size],\n",
    "                                                    y: Y_test[i:i+test_batch_size],\n",
    "                                                    keep_prob: 1,\n",
    "                                                    phase: False})\n",
    "        else:\n",
    "            test_loss, test_acc = sess.run([cost, accuracy], \n",
    "                                            feed_dict={x: X_test[i:],\n",
    "                                                       y: Y_test[i:],\n",
    "                                                       keep_prob: 1,\n",
    "                                                       phase: False})\n",
    "   \n",
    "        total_test_loss = total_test_loss+test_loss\n",
    "        total_test_acc = total_test_acc+test_acc\n",
    "        count+=1\n",
    "        \n",
    "        i+=test_batch_size\n",
    "             \n",
    "    avg_test_loss = total_test_loss/count # Average test loss\n",
    "    avg_test_acc = total_test_acc/count # Average test accuracy\n",
    "    \n",
    "    print \"Test Loss = \" + \\\n",
    "          \"{:.3f}\".format(avg_test_loss) + \", Test Accuracy = \" + \\\n",
    "          \"{:.3f}%\".format(avg_test_acc*100)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Saving logs about change of training and validation loss and accuracy over epochs in another file.\n",
    "\n",
    "import h5py\n",
    "\n",
    "file = h5py.File('Training_logs.h5','w')\n",
    "file.create_dataset('val_acc', data=np.array(val_acc_list))\n",
    "file.create_dataset('val_loss', data=np.array(val_loss_list))\n",
    "file.create_dataset('acc', data=np.array(acc_list))\n",
    "file.create_dataset('loss', data=np.array(loss_list))\n",
    "\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import h5py\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "log = h5py.File('Training_logs.h5','r+') # Loading logs about change of training and validation loss and accuracy over epochs\n",
    "\n",
    "y1 = log['val_acc'][...]\n",
    "y2 = log['acc'][...]\n",
    "\n",
    "x = np.arange(1,len(y1)+1,1) # (1 = starting epoch, len(y1) = no. of epochs, 1 = step) \n",
    "\n",
    "plt.plot(x,y1,'b',label='Validation Accuracy') \n",
    "plt.plot(x,y2,'r',label='Training Accuracy')\n",
    "plt.legend(loc='lower right')\n",
    "plt.xlabel('epoch')\n",
    "plt.show()\n",
    "\n",
    "y1 = log['val_loss'][...]\n",
    "y2 = log['loss'][...]\n",
    "\n",
    "plt.plot(x,y1,'b',label='Validation Loss')\n",
    "plt.plot(x,y2,'r',label='Training Loss')\n",
    "plt.legend(loc='upper right')\n",
    "plt.xlabel('epoch')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}