Q-Learning with Neural Networks in OpenAI Gym Environment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Q-Learning with Neural Networks in OpenAI Gym Environment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load OpenAI Gym \"FrozenLake\" Environment "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2018-01-26 22:41:11,475] Making new env: FrozenLake-v0\n"
     ]
    }
   ],
   "source": [
    "env = gym.make(\"FrozenLake-v0\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Implementing Q-Learning algorithm with Neural Network approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Let's construct a feed forward network which takes one-hot encoded input states as input and outputs the actions to be taken\n",
    "inputs = tf.placeholder(shape=[1,16], dtype=tf.float32)\n",
    "W = tf.Variable(tf.random_uniform([16,4], minval=0, maxval=0.01))\n",
    "Qout = tf.matmul(inputs, W)\n",
    "predict = tf.argmax(Qout, 1)\n",
    "\n",
    "#The objective function is minimized by computing the loss which is the sum of square difference between the target and prediction Q values\n",
    "nextQ = tf.placeholder(shape=[1,4], dtype=tf.float32)\n",
    "loss = tf.reduce_sum(tf.square(nextQ - Qout))\n",
    "trainer = tf.train.GradientDescentOptimizer(learning_rate=0.1)\n",
    "updateModel = trainer.minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.,  1.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,  0.,\n",
       "         0.,  0.,  0.]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.identity(16)[1:2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Training the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "init = tf.global_variables_initializer()\n",
    "\n",
    "#learning parameters\n",
    "y = 0.99\n",
    "e = 0.1\n",
    "num_episodes = 2000\n",
    "#create lists to contain total rewards and steps per episode\n",
    "jList = []\n",
    "rList = []\n",
    "with tf.Session() as sess:\n",
    "    sess.run(init)\n",
    "    for i in range(num_episodes):\n",
    "        #On every iteration through the episode we reset environment and get first new observation\n",
    "        s = env.reset()\n",
    "        rAll = 0\n",
    "        d = False\n",
    "        j = 0\n",
    "        #The Q-Network\n",
    "        while j < 99:\n",
    "            j += 1\n",
    "            #Choose an action greedily(with e chance of random function) from the Q-Network\n",
    "            a, allQ = sess.run([predict, Qout], feed_dict={inputs: np.identity(16)[s:s+1]})\n",
    "            if(np.random.rand(1) < e):\n",
    "                a[0] = env.action_space.sample()\n",
    "            #Get new state and reward from the environment\n",
    "            s1, r, d, _ = env.step(a[0])\n",
    "            #We get the Q-values by feeding the new state through our network\n",
    "            Q1 = sess.run(Qout, feed_dict={inputs: np.identity(16)[s1:s1+1]})\n",
    "            #Obtain maxQ and set our target value for choosen action\n",
    "            maxQ1 = np.max(Q1)\n",
    "            targetQ = allQ\n",
    "            targetQ[0,a[0]] = r + y*maxQ1\n",
    "            #Train our network using target and predicted Q values\n",
    "            _, W1 = sess.run([updateModel, W], feed_dict={inputs: np.identity(16)[s:s+1], nextQ: targetQ})\n",
    "            rAll += r\n",
    "            s = s1\n",
    "            if d == True:\n",
    "                #Reduce chance of random action as we train the model\n",
    "                e = 1./((i/50) + 10)\n",
    "                break\n",
    "        jList.append(j)\n",
    "        rList.append(rAll)\n",
    "\n",
    "print(\"Percent of succesful episodes: \" + str(sum(rList)/num_episodes) + \"%\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We observe that the FrozenLake problem when solved through neural networks isnt efficient when compared to Q-Table approach.(This can improved by using methods such as Experience Replay and Freezing Target Networks."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}