hill-climbing/Hill_Climbing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hill Climbing\n",
    "\n",
    "---\n",
    "\n",
    "In this notebook, we will train hill climbing with adaptive noise scaling with OpenAI Gym's Cartpole environment."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Import the Necessary Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pyvirtualdisplay in /opt/conda/lib/python3.6/site-packages (1.3.2)\r\n",
      "Requirement already satisfied: EasyProcess in /opt/conda/lib/python3.6/site-packages (from pyvirtualdisplay) (0.3)\r\n"
     ]
    }
   ],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "!python -m pip install pyvirtualdisplay\n",
    "from pyvirtualdisplay import Display\n",
    "display = Display(visible=0, size=(1400, 900))\n",
    "display.start()\n",
    "\n",
    "is_ipython = 'inline' in plt.get_backend()\n",
    "if is_ipython:\n",
    "    from IPython import display\n",
    "\n",
    "plt.ion()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Define the Policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
      "observation space: Box(4,)\n",
      "action space: Discrete(2)\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "print('observation space:', env.observation_space)\n",
    "print('action space:', env.action_space)\n",
    "\n",
    "class Policy():\n",
    "    def __init__(self, s_size=4, a_size=2):\n",
    "        self.w = 1e-4*np.random.rand(s_size, a_size)  # weights for simple linear policy: state_space x action_space\n",
    "        \n",
    "    def forward(self, state):\n",
    "        x = np.dot(state, self.w)\n",
    "        return np.exp(x)/sum(np.exp(x))\n",
    "    \n",
    "    def act(self, state):\n",
    "        probs = self.forward(state)\n",
    "        #action = np.random.choice(2, p=probs) # option 1: stochastic policy\n",
    "        action = np.argmax(probs)              # option 2: deterministic policy\n",
    "        return action"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Train the Agent with Stochastic Policy Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
      "Episode 100\tAverage Score: 175.24\n",
      "Environment solved in 13 episodes!\tAverage Score: 196.21\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "env.seed(0)\n",
    "np.random.seed(0)\n",
    "\n",
    "policy = Policy()\n",
    "\n",
    "def hill_climbing(n_episodes=1000, max_t=1000, gamma=1.0, print_every=100, noise_scale=1e-2):\n",
    "    \"\"\"Implementation of hill climbing with adaptive noise scaling.\n",
    "        \n",
    "    Params\n",
    "    ======\n",
    "        n_episodes (int): maximum number of training episodes\n",
    "        max_t (int): maximum number of timesteps per episode\n",
    "        gamma (float): discount rate\n",
    "        print_every (int): how often to print average score (over last 100 episodes)\n",
    "        noise_scale (float): standard deviation of additive noise\n",
    "    \"\"\"\n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores = []\n",
    "    best_R = -np.Inf\n",
    "    best_w = policy.w\n",
    "    for i_episode in range(1, n_episodes+1):\n",
    "        rewards = []\n",
    "        state = env.reset()\n",
    "        for t in range(max_t):\n",
    "            action = policy.act(state)\n",
    "            state, reward, done, _ = env.step(action)\n",
    "            rewards.append(reward)\n",
    "            if done:\n",
    "                break \n",
    "        scores_deque.append(sum(rewards))\n",
    "        scores.append(sum(rewards))\n",
    "\n",
    "        discounts = [gamma**i for i in range(len(rewards)+1)]\n",
    "        R = sum([a*b for a,b in zip(discounts, rewards)])\n",
    "\n",
    "        if R >= best_R: # found better weights\n",
    "            best_R = R\n",
    "            best_w = policy.w\n",
    "            noise_scale = max(1e-3, noise_scale / 2)\n",
    "            policy.w += noise_scale * np.random.rand(*policy.w.shape) \n",
    "        else: # did not find better weights\n",
    "            noise_scale = min(2, noise_scale * 2)\n",
    "            policy.w = best_w + noise_scale * np.random.rand(*policy.w.shape)\n",
    "\n",
    "        if i_episode % print_every == 0:\n",
    "            print('Episode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_deque)))\n",
    "        if np.mean(scores_deque)>=195.0:\n",
    "            print('Environment solved in {:d} episodes!\\tAverage Score: {:.2f}'.format(i_episode-100, np.mean(scores_deque)))\n",
    "            policy.w = best_w\n",
    "            break\n",
    "        \n",
    "    return scores\n",
    "            \n",
    "scores = hill_climbing()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Plot the Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XuUnXV97/H3d1/mknsgkxBCYkABQWujnVJbLwfFWrEuqZ5apLZQ62m09XqOXS3ac6p2LdfqsV5aV3uosVDw1KZYo5XTcqwUPWIvUoPQiNwEBAkMySSBJDPJzOzL9/zxPM+eZ/Y8k5k94blMns9rrVmz59l7z/w2O/y++/v7/i7m7oiIiHSr5N0AEREpJgUIERFJpAAhIiKJFCBERCSRAoSIiCRSgBARkUQKECIikkgBQkREEilAiIhIolreDTgZ69at861bt+bdDBGRJeWOO+444O5D8z1uSQeIrVu3snv37rybISKypJjZowt5nIaYREQkkQKEiIgkUoAQEZFEChAiIpJIAUJERBKlFiDMbLOZfcPM7jWz75vZe8Prp5nZLWb2g/D72vC6mdmnzexBM9tjZi9Kq20iIjK/NDOIJvB+d78AeDHwTjO7ELgauNXdzwVuDX8GuBQ4N/zaDlyTYttERGQeqa2DcPcRYCS8fdTM7gU2AZcBF4cPuwH4f8Dvhtc/58EZqN82szVmtjH8Pae0f33wAGesHuCcoRULfs6xqSbX/+sjTEy1UmyZiBTVeWes5HUvODPVv5HJQjkz2wq8ELgd2BB1+u4+Ymbrw4dtAh6LPW1veG1GgDCz7QQZBlu2bEm13Vn5nV17+Jlnn87HfvHHF/yc2x8+xMe+ej8AZmm1TESK6nUvOHPpBwgzWwHsAt7n7kds7t4s6Q6fdcF9B7ADYHh4eNb9S9Fks83xRrvH5wSZw83veRkXnrkqjWaJSMmlOovJzOoEweHz7v6l8PI+M9sY3r8R2B9e3wtsjj39LOCJNNtXFK2202j2FiCmWkFsrFeVPohIOtKcxWTAtcC97v7J2F03AVeFt68CvhK7fmU4m+nFwOEy1B8AGq02U63eAkQzfHytqpnKIpKONIeYXgL8KvA9M7srvPZB4A+BL5jZ24AfAW8K77sZeC3wIHAMeGuKbSuUVttp9BwgggyiVlEGISLpSHMW0z+TXFcAuCTh8Q68M632FFmz7Uz2OMTUaAePryuDEJGUqHcpgJPKIFSDEJGUlDJAHJlocNsDoxwcm8y7Kbg7rbYz1WsGEQaUeqWUb6GIZKCUvcvDo+Nced2/8x97n867KTTbQSbQcwbRVgYhIukqZYCICruNVv7LKFqdANFbW6JZTKpBiEhaStm7RJ1qswABIsoceh9i0joIEUlXKQNENcwgmu3eOuU0RBlEr+sgGq021YpxgpXpIiInpZQBIvrUXYQMIqol9JpBNNuuNRAikqpSBoho9XGRMohei9SNVlv1BxFJVSl7mHqBitSLrUE0W64ZTCKSqlIGiE4G0eOn9jREGUSz7bTbCw9YzXabmtZAiEiKStnDRJ+8mz10yGmJt6HRw5BXo+WawSQiqSplgIhWHxdhiCleKO9lmKmpGoSIpKyUPUwngyjAEFO8UN5LwGq0VYMQkXSVM0BEReoCDDG12ovLIBrNtvZhEpFUlbKHMTNqFStIBhGrQfTQnqYyCBFJWSkDBASrqQtRpI4NK/VyJkSj1dZpciKSqjSPHL3OzPab2d2xazea2V3h1yPRSXNmttXMjsfu+/O02hWpVysFWUkdr0H0UqT2znoOEZE0pHnk6PXAnwKfiy64++XRbTP7BHA49viH3H1biu2ZoVa1Qq2khh5nMbXbGmISkVSllkG4+23AoaT7LNhh7peAnWn9/fnUKpViTHNdZA0iWAehISYRSU9ePczLgH3u/oPYtbPN7E4z+6aZvSztBtSrBSlSx9dB9FSk1joIEUlXmkNMJ3IFM7OHEWCLux80s58A/s7MnufuR7qfaGbbge0AW7ZsWXQDgiGm/DOIVmyYq7eFctrNVUTSlflHUDOrAW8Eboyuufukux8Mb98BPAScl/R8d9/h7sPuPjw0NLTodtQrlZ53UE3DzCGmHhbKaSW1iKQsjx7mVcB97r43umBmQ2ZWDW+fA5wLPJxmI2pVK8YspkVutdHQbq4ikrI0p7nuBP4NON/M9prZ28K73szs4vTLgT1m9h/AF4F3uHtigfuZUqtUCjGLadEL5VrazVVE0pVaDcLdr5jj+q8lXNsF7EqrLUnqVSvELKbF1iAabe3mKiLpKu1H0GAldbEyiJ5mMbW0DkJE0lXaAFErykrq1mKHmFxDTCKSqtL2MPWCTHNtLnY313ZbQ0wikqrSBohapVKIhXKtk9mLSdNcRSRFpe1hilKkbiximqu7h9t9l/btE5EMlLaHKco012izvmrFmFpgwIqCinZzFZE0lTdAFGWhXBggBuvVBWcQUWBTBiEiaSptD1OvVmgUIoNoU6sYfbWFb/3RySBUpBaRFJU2QARHjhYgg2g51YpRr9rCM4gwkGizPhFJU3kDRLU450H0mkFEw1IaYhKRNJW2h6kVZCV1qx1lEJUFr6SOAomGmEQkTeUNEFWjVYAMItq2u69a6WGIKapBlPbtE5EMlLaHKU6ROsggehti0iwmEUlfaXuYwhSpwxpEb0NMWgchIukrb4CoVmi2Hfd8g0QrXBHdV63QaCa3ZaLRYqLR6vwcZRrKIEQkTaXtYaJP33lv2NdoBesg6rUKk3NkEL+7aw/v3nln7DnRLCZlECKSntQODCq66NN3sOldfu3o1CCqFRpzFKkfPXhsxoyraB1EXdt9i0iK0jxy9Doz229md8eufdjMHjezu8Kv18bu+4CZPWhm95vZz6XVrkg0RTTvQnWzU6S2OYvU45NNJhvtGc8BZRAikq40P4JeD7wm4fqn3H1b+HUzgJldSHBW9fPC5/wvM0v1c320CjnvQnWr7dPTXE8UIJqztwXXOggRSVNqAcLdbwMOLfDhlwF/4+6T7v5D4EHgorTaBvEhpnwziEar3VkoN9cQ0/hUa8YaCa2DEJEs5NHDvMvM9oRDUGvDa5uAx2KP2RteS02UQTRyLlK3ommuteQMwt3DDGJ6FlNnHYRqECKSoqx7mGuAZwPbgBHgE+H1pLGSxJ7bzLab2W4z2z06OrrohkQZRN6rqYODf2zOldSTzTbNtncNMWk3VxFJX6YBwt33uXvL3dvAZ5keRtoLbI499CzgiTl+xw53H3b34aGhoUW3pTBF6labWqVC3xwZxPhkEyCxBqF1ECKSpkx7GDPbGPvxDUA0w+km4M1m1m9mZwPnAv+eZlui4ZkiFKk701wT2jI+2eo8LqqXRG3Wdt8ikqbU1kGY2U7gYmCdme0FPgRcbGbbCIaPHgHeDuDu3zezLwD3AE3gne7eSvq9z5RoiuhC9z9KS3yrjVbbOwEjMhZmEABTrXawTXk7msWkDEJE0pNagHD3KxIuX3uCx38U+Gha7ekWDTHlvZK6FdUgakFnH8xqmp7hOz41HSAmG22W9cUyCNUgRCRFpf0IOj3ElP8011ql0glY3XWIeAYR1SEaWkktIhkobQ8zPcSUfwYRbfcNzJrJNB4fYgrv00pqEclCaQNENH6f96ly8WmuMLsmMj4jgwjKMp29mFSDEJEUlbaHKcpWG83WdJEaZmcQY5PTtfrpISatgxCR9JU2QNTn+MSetWCzvsqMInVcYgbRDrbnMFOAEJH0lDZARFNJW7nPYmrPyCAmT1CDiGcQWgMhImkrbYCYXkldjK02+jsZxMz2zDWLSfUHEUlbaXuZokxz7a5BnHCIqTG9klozmEQkbeUNENViFKlbYQ2isw4ioUgd1SfiNQjt5CoiaSttL9P5xJ77NNegBtFZB5GQQZy+vC+4L1aD0AwmEUlbaQNEEaa5tttO24NsZq5pruNTTdYuCwJEVINoqgYhIhkobS9TK8A0186K6Eq8SD17q43TV8wMEI22ahAikr7SBogibNYXTbENahBzb7Vx2vIoQEyvpNY+TCKSttL2MkWYxdTsbNttJ1go1+oMMcVrEMogRCRtJQ4Q+W/WF9U/qvGtNmLtcXfGp5qsHKjRV63MWAeh0+REJG2l7WUqFaNi+a6kjtcgknZzPTbVwh2W99for1VmrIOoayW1iKSstAECmHE6Wx7iNYik3VyjRXLL+2v01ysz10FoiElEUpZagDCz68xsv5ndHbv2R2Z2n5ntMbMvm9ma8PpWMztuZneFX3+eVrvi6hXLdZprVIMIprnOXigXbbOxor/aNcTkmuYqIqlLs5e5HnhN17VbgOe7+wuAB4APxO57yN23hV/vSLFdHbVqJd8idWt6iKlWrVCx7gwiyBiW99Xor1djBwZpHYSIpC+1XsbdbwMOdV37mrtHmwt9Gzgrrb+/EPWq5bpZX7M9XaQO2lOZI4MIaxCdaa7azVVE0pfnx9BfB/5v7OezzexOM/ummb1srieZ2XYz221mu0dHR0+qAbVKvhlEVIOIsoG+WmXGVhszahC1inZzFZFM5dLLmNnvAU3g8+GlEWCLu78Q+G/AX5vZqqTnuvsOdx929+GhoaGTaketmm8NIhpOijKIvq4MYnwqHiCq07OYtJJaRDKQeYAws6uA1wFvcXcHcPdJdz8Y3r4DeAg4L+221KuVXIeYWrFprhBkEPEaRHyIKZ5dNJrazVVE0pdpL2NmrwF+F3i9ux+LXR8ys2p4+xzgXODhtNtTq1jOK6ln1yDiC/emh5iqM2oQjbZ2cxWR9NXS+sVmthO4GFhnZnuBDxHMWuoHbgnPU/52OGPp5cAfmFkTaAHvcPdDib/4GVTr6pCz1l2DqFetq0gdn8UUXyindRAikr7UAoS7X5Fw+do5HrsL2JVWW+ZSqxitHBfKNbtrELXqrCL1sr4qlYoFNYjm9EpqDTGJSNpK3cvUqpbrbq7N7hpEVwYxPtlkeX8tvG+6gN1otzXEJCKpK3WAqFcquZ4H0SlSx6a5dhepV4QBYsZWG1pJLSIZKHUvU5RprrU5FsoFGUQVoLMOwt3Daa6lfutEJAOl7mVqBZnmWp1jmuv4ZIvlfWEGEdYgomEp7eYqImlbcIAws5ea2VvD20NmdnZ6zcpGvSDTXGdkELGMZmwyOAsCguDRajsTjWCYSRmEiKRtQb2MmX2IYP1CtLleHfirtBqVlbyHmGbVIKoVpsI6AwQrqaMidXRmdbSBn4rUIpK2hX4MfQPwemAcwN2fAFam1ais5H0eRHcNIhhimrlQrjtARKurtVmfiKRtoQFiKtwWwwHMbHl6TcpO3udBdNcgZi+Ui89iCorVx8L9mTTEJCJpW2gv8wUz+wywxsx+A/gn4LPpNSsbuZ8H0Rliml2kbrbaTDTasSL1zAxCQ0wikrYFraR294+b2c8CR4Dzgd9391tSbVkGapV8F8pNb9YXbbUxPc11fCrcZiOc5to3qwahDEJE0jVvgAg30ftHd38VwYlwp4y8V1LP2u47tmPreGwnVwimucava4hJRNI2by/j7i3gmJmtzqA9maoVZSV1/DyIVrAYLn5YECQMMalILSIpW+hmfRPA98zsFsKZTADu/p5UWpWRes7TXGfVIKoV3IPAMTYrg4iGmJRBiEg2Fhog/iH8OqXUqhWaue7m2lWDCINAo+WdWkNns75ZAUIZhIika6FF6hvMrI/pU97ud/dGes3KRr1iNFqOuxOeT5GpaKvxaLQoKjxPNdudDGJ6L6bge3RGRF3bfYtIyhYUIMzsYuAG4BHAgM1mdpW735Ze09IXDdO0cjrjudl2ahXrBKcoS5hqtWcXqevKIEQkWwsdYvoE8Gp3vx/AzM4DdgI/kVbDshB1ssHuqNn//e7A1Bfenmq1GZ9KLlJH17UOQkTSttBxinoUHADc/QGC/ZhOyMyuM7P9ZnZ37NppZnaLmf0g/L42vG5m9mkze9DM9pjZi3p9Mb2KhmnymsnU6DoZLsogGs0239t7mJX9NVYP1mfc18kgNMQkIilbaC+z28yuNbOLw6/PAncs4HnXA6/punY1cKu7nwvcGv4McClwbvi1HbhmgW1btE4GkdNMpla73VkDAdM1iPGpJl+7Zx+vunBD59r0OggtlBORbCy0l/lN4PvAe4D3AvcA75jvSWGN4lDX5csI6hmE338hdv1zHvg2wbYeGxfYvkWJ1h/ktViu2fYZQ0V9Yad/2wMHOHy8waXPP6Nzn7baEJGsLbQGUQP+xN0/CZ3V1f2L/Jsb3H0EwN1HzGx9eH0T8FjscXvDayPxJ5vZdoIMgy1btiyyCYGoSJ3XVNdmy2dmEGEQ+Mpdj7O8r8rLzxvq3Nddg9A6CBFJ20J7mVuBwdjPgwQb9j2Tkj4Sz/po7+473H3Y3YeHhoYSnrJwnQwipyGmYBZTrAYRdvr3PXmUSy7YwEB9unJuZvRVK7EahDIIEUnXQgPEgLuPRT+Et5ct8m/ui4aOwu/7w+t7gc2xx50FPLHIv7Eg0Th+XkXq7hpEVIgGeO2PzR5d669VYkNMyiBEJF0L7WXG47OKzGwYOL7Iv3kTcFV4+yrgK7HrV4azmV4MHI6GotISn+aah2bXNNeo01/WV+Xi82dnR/31ChON8JAh1SBEJGULrUG8D/hbM3uCYNjnTODy+Z5kZjuBi4F1ZrYX+BDwhwTnS7wN+BHwpvDhNwOvBR4EjgFvXfjLWJxaztNcmy2fMVQUDTG98rnrZwwvRfpjizW0klpE0nbCAGFmPwk85u7fMbPnAm8H3gh8FfjhfL/c3a+Y465LEh7rwDvnbfEzqJ7zNNdm26nGOvr1q/oZqFd40/DmxMf3x4aglEGISNrm+xj6GWAqvP3TwAeBPwOeAnak2K5M5D2LqdVuz5iuum5FP3d/+Of4T+clF9/jNQrVIEQkbfMNMVXdPVrHcDmww913AbvM7K50m5a+6EyFRq4ZxMxM4ETTV/tnBAhlECKSrvk+hlbNLAoilwBfj9230PpFYXUyiLwCRFcNYj5RDaIa2+BPRCQt83XyO4FvmtkBgllL3wIws+cAh1NuW+qqnZXUeQ0xzc4gTiTa0VVrIEQkCycMEO7+UTO7FdgIfC0sJEOQebw77calLf8idZvl9YUnYtEsJ9UfRCQL8/ZO4b5I3dceSKc52Yqmuea21cZiMwjVH0QkA6X+KBplELkVqRdZg9BW3yKShVL3NPlPc/WeOvtoFpNmMIlIFsodIHKf5tqm2kNn31dTDUJEslPqnqae9zTXdq9DTKpBiEh2Sh0gpjfrK8Z5EPOJahDah0lEslDqnmb6TOq8jhz1njp7ZRAikqVSB4jpM6mDDOLqXXv4g/9zT2Z/f7E1CJ0mJyJZWPLbZZyMateZ1Lf/8BBDKxd7kmrvFluDqGsltYhkoNQfRbuL1KNHJ5lotDL7+61eaxDhGREaYhKRLJQ6QASb3gVDPcemmoxNNjMNEM229zRltV/TXEUkQ5kPMZnZ+cCNsUvnAL8PrAF+AxgNr3/Q3W9Ouz31SoVGyzlwNDj24nimAaK9qFlM2qxPRLKQeYBw9/uBbQBmVgUeB75McMTop9z941m2p1Y1mq02o2MTAJ0zn7PQaw1CC+VEJEt59zSXAA+5+6N5NaBWMZptZ/ToJAATU9lkEO22497bvkoaYhKRLOXd07yZ4MyJyLvMbI+ZXWdma7NoQL1aodFqTweIZjYBohEuzuul4Kx1ECKSpdwChJn1Aa8H/ja8dA3wbILhpxHgE3M8b7uZ7Taz3aOjo0kP6UkwxOTsDwNEo+WddRFpaoVTaxc1i0krqUUkA3n2NJcC33X3fQDuvs/dW+7eBj4LXJT0JHff4e7D7j48NDR00o2oVSo02tMZBMBEM/0AEa296KkG0TkwSBmEiKQvzwBxBbHhJTPbGLvvDcDdWTSiHmYQ8QBxPIM6RKvVe4DQgUEikqVcVlKb2TLgZ4G3xy5/zMy2AQ480nVfaqoVo9V2RsdiGUQGU12jGkR1EesgNMQkIlnIJUC4+zHg9K5rv5pHW+JF6sF6leONViYBorWIIabObq7KIEQkA6X/KFqrGo1WmwNjk2w+bRDIZrFccxFDTFoHISJZKn1PU6tUODA2RaPlbDltGZDNYrlOBrGoaa6lf9tEJAOl72nqVWPk8HEANocBIpMMIqpB9LhQ7tUXbmD4WZksERGRkiv1dt8wnUEAbF4bZRBZBIjeh5jMjB1XDqfVJBGRGUqfQcSHeKaHmIpZgxARyVLpA0S84Ls5ywCxiBqEiEiWSh8gok/wA/UK61b0ARktlFtEDUJEJEul752iDGJoZT/L+oKSTCZbbYRDTDo+VESKqvQBItosb2hFf2caaTYZRO+b9YmIZKn0ASKqAaxfOUClYvTXKpls+d1QDUJECq70AaJemR5iAhioVzM5NEg1CBEputL3TtEn+ChADNarmayk1jRXESm60geIeJEagtlM2ayk1hCTiBRb6QNELVakhnCIqaArqUVEsqQAMSuDqGaSQUQ1CJ3tICJFVfreqZ5Qg5jMsAahaa4iUlSlDxCrB+sM1qusW6EahIhIXG67uZrZI8BRoAU03X3YzE4DbgS2Ehw7+kvu/lSa7XjLTz2LVz53fecwnsG+bGsQyiBEpKjyziBe4e7b3D3aw/pq4FZ3Pxe4Nfw5VYN9Vc4ZWtH5eaCWUQ2iFQxj1VWDEJGCKlrvdBlwQ3j7BuAXsm7AQF9G6yCiDEJDTCJSUHkGCAe+ZmZ3mNn28NoGdx8BCL+vz7pRAzVNcxURgXxPlHuJuz9hZuuBW8zsvoU8KQwm2wG2bNnyjDdqsK+SSYDQZn0iUnS5ZRDu/kT4fT/wZeAiYJ+ZbQQIv+9PeN4Odx929+GhoaFnvF0DtSrNttNopTvMNL3dd9FG+UREArn0Tma23MxWRreBVwN3AzcBV4UPuwr4StZtG+yrAumfKtdstzGDijIIESmovIaYNgBfNrOoDX/t7l81s+8AXzCztwE/At6UdcP660GAON5osXKgntrfabZd9QcRKbRcAoS7Pwz8eML1g8Al2bdo2mAYINJeTd1qu7bZEJFCUw/VZaAeniqX9hBTSxmEiBSbAkSXKIPIogahNRAiUmQKEF0GohpEyqfKqQYhIkWnANElChATzXRrEEcnmqzoz3MZiojIiSlAdOnUIFLOIA6OTXJ6uIOsiEgRKUB06cxiaqYdIKY4fXlfqn9DRORkKEB0yaoGcXBcGYSIFJsCRJfB2EK5tLTbzqHxKdatUAYhIsWlANGlU6ROcaHc08cbtB1O0xCTiBSYAkSX/lr6C+UOjk0CaIhJRApNAaJLpWL01ypMnmSAuO2BUW57YDTxvgNjUwCsUwYhIgWmifgJBvtO/tjRP/rH+6kYvPy82VuSHxoPAoQyCBEpMmUQCQbryafK3f/kUb5+374F/Y6RwxPsOzKZeN/B8WiISRmEiBSXAkSCgXqV4wlF6j/9xoP89t/umff5U802B8YmGR2b7JwcF3dgbAozWLtMAUJEiksBIsHAHBnEyNPHOTQ+xbGp5gmfv+/IBBBs6R1lC3EHxyZZu6xPx42KSKEpQCQYqCefSz1yOOj4n3j6+Amf/2QYIAD2JwwzHRrXKmoRKT4FiARJNYhW2zuZwd6nThwgokAC09lE3MGxKdUfRKTwMg8QZrbZzL5hZvea2ffN7L3h9Q+b2eNmdlf49dqs2xYJahAzA8TBsUmaYT3h8fkyiMPT9ycVqg+MT3L6cs1gEpFiy2OaaxN4v7t/18xWAneY2S3hfZ9y94/n0KYZggxiZpE6nhXMN8T0xNMTwe9otpRBiMiSlXmAcPcRYCS8fdTM7gU2Zd2OE+mvV2Zt1hcPEI/PM8T05OEJNq0d5OljDfYfnRkgGq02h483lEGISOHlWoMws63AC4Hbw0vvMrM9Znadma2d4znbzWy3me0eHU1eqXyyBuvVWdt9R8NGz1m/Yt4hppEjE2xcPcCGVf2zhpie6iySUwYhIsWWW4AwsxXALuB97n4EuAZ4NrCNIMP4RNLz3H2Huw+7+/DQ0OxVys+EgXp1dgZxZIK+aoUf27SaJ56ePWwU9+Th45yxaoANqwZmDTF1ttlQgBCRgsslQJhZnSA4fN7dvwTg7vvcveXubeCzwEV5tA3CGkSzjfv0IrcnD09wxuoBzlo7yJNHJmi2knd7bbTa7D86OWcGEa2LOE1DTCJScHnMYjLgWuBed/9k7PrG2MPeANydddsiA/UKrbbTaE0HiJGngwCxac0grbbPWOsQN3p0EnfYuGaQ9SsHODg+SSMWTA6OaYhJRJaGPGYxvQT4VeB7ZnZXeO2DwBVmtg1w4BHg7Tm0DYidCdFs0Rdu/z1y5Dgv2rKWM9cMAkGh+qy1y2Y9Nypmn7F6AHdwhwNjk2xcHTzv4Hi0k6syCBEptjxmMf0zkLTHxM1Zt2UunQAx1WLVQJ1229l3OOjkN60NOvonDicXqp8MA8TG1QO0w3UT+47EAsTYJLWKsWpQG+mKSLGpl0ow2HWq3KFjU0y12mwMh5hg7qmuI2Hg2LhqkGYrChDTw1HRGohgpE1EpLgUIBIMdJ1L/WRs2GigXmXdir45p7qOHA4Wya0arLG+FQwj7Y8HiPFJFahFZEnQXkwJBvuC/yzRfkwjsWEjgDPXDPL4HFNdnzw8wcY1A5gZpy/vp1qxGTOZDoxNaYqriCwJChAJBmozM4ho2OiMMEBsWjPI408dS3zuyOHjnUBSrRhDK/pnDDFpJ1cRWSoUIBIM9EU1iOkMolaxzsyjTWsGefzp4zPWSUSePDzBGasGOz9vWNXPvqPTGcTBsUkdNSoiS4ICRIIog5iI1SA2rBqgEh7ws2ntIBONduds6Uir7ewLF8lF1q8a6NQgjk+1GJ9qaQ2EiCwJChAJBvtmzmIaOXycM9dMd/rRWojuLTcOhEeMnhELEMFq6uBxnbOoNcQkIkuAAkSCgXrwnyU+i+mM1dPDRp2prk/PrENExex4MNmwcoCnjjWYbLY6GYd2chWRpUABIsGyejD790eHjuHujByemDFsdFa4WK77ZLlox9eZNYjgefuPTHLfyFFA22yIyNKgdRAJVi8au0UWAAAIRklEQVSr86oLNvCZbz7EOeuWM9lsc8aq6QCxerDO8r4qX/ru42xaM8grnrue7zxyiBv+9VGArhpEkC380737+NhX7+f5m1bxvDNXZ/uCREQWQQFiDp++Yhtv3vFtfmfXHmBmp29mvP/V5/OZ2x7iNz//XaoVo9V2Vg/W+c2Ln83aWI0hyiD+4O/vYdOaQa77tZ/s7O8kIlJkChBzWNZX49qrfpI3XvMvPHbo+IzCM8Cvv/RsrvzpZ/GtBw/wzftHGd66llddsKGzCjsSBYhVA3Wuf+tFrF858/eIiBSVAsQJDK3s53O//lP87397lAvPXDXr/lq1wivOX88rzl8/5+9Yu6zOu1/5HC65YAPPWb8izeaKiDyjLGmx11IxPDzsu3fvzrsZIiJLipnd4e7D8z1Og+EiIpJIAUJERBIVLkCY2WvM7H4ze9DMrs67PSIiZVWoAGFmVeDPgEuBCwmOIb0w31aJiJRToQIEcBHwoLs/7O5TwN8Al+XcJhGRUipagNgEPBb7eW94TUREMla0AJF0UPOMebhmtt3MdpvZ7tHR0YyaJSJSPkULEHuBzbGfzwKeiD/A3Xe4+7C7Dw8NDWXaOBGRMinUQjkzqwEPAJcAjwPfAX7Z3b8/x+NHgUd7/DPrgAMn086COlVfF5y6r02va+k5VV7bs9x93k/Yhdpqw92bZvYu4B+BKnDdXMEhfHzPKYSZ7V7ICsKl5lR9XXDqvja9rqXnVH5tSQoVIADc/Wbg5rzbISJSdkWrQYiISEGUMUDsyLsBKTlVXxecuq9Nr2vpOZVf2yyFKlKLiEhxlDGDEBGRBShNgDiVNgE0s81m9g0zu9fMvm9m7w2vn2Zmt5jZD8Lva/Nu62KYWdXM7jSzvw9/PtvMbg9f141m1jff7ygaM1tjZl80s/vC9+2nT6H367+G/w7vNrOdZjawFN8zM7vOzPab2d2xa4nvkQU+HfYne8zsRfm1PD2lCBCn4CaATeD97n4B8GLgneHruRq41d3PBW4Nf16K3gvcG/v5fwKfCl/XU8DbcmnVyfkT4Kvu/lzgxwle35J/v8xsE/AeYNjdn08wPf3NLM337HrgNV3X5nqPLgXODb+2A9dk1MZMlSJAcIptAujuI+7+3fD2UYLOZhPBa7ohfNgNwC/k08LFM7OzgJ8H/iL82YBXAl8MH7LkXpeZrQJeDlwL4O5T7v40p8D7FaoBg+FC12XACEvwPXP324BDXZfneo8uAz7ngW8Da8xsYzYtzU5ZAsQpuwmgmW0FXgjcDmxw9xEIgggw92HZxfXHwO8A7fDn04Gn3b0Z/rwU37tzgFHgL8Ohs78ws+WcAu+Xuz8OfBz4EUFgOAzcwdJ/zyJzvUenbJ8SV5YAMe8mgEuRma0AdgHvc/cjebfnZJnZ64D97n5H/HLCQ5fae1cDXgRc4+4vBMZZgsNJScIx+cuAs4EzgeUEwy/dltp7Np9T4d/lvMoSIObdBHCpMbM6QXD4vLt/Kby8L0pzw+/782rfIr0EeL2ZPUIwDPhKgoxiTTh8AUvzvdsL7HX328Ofv0gQMJb6+wXwKuCH7j7q7g3gS8DPsPTfs8hc79Ep16ckKUuA+A5wbjizoo+giHZTzm1atHBc/lrgXnf/ZOyum4CrwttXAV/Jum0nw90/4O5nuftWgvfo6+7+FuAbwC+GD1uKr+tJ4DEzOz+8dAlwD0v8/Qr9CHixmS0L/11Gr21Jv2cxc71HNwFXhrOZXgwcjoaiTiWlWShnZq8l+DQabQL40ZybtGhm9lLgW8D3mB6r/yBBHeILwBaC/3Hf5O7dRbclwcwuBn7b3V9nZucQZBSnAXcCv+Luk3m2r1dmto2g8N4HPAy8leAD2pJ/v8zsI8DlBLPr7gT+C8F4/JJ6z8xsJ3AxwY6t+4APAX9HwnsUBsM/JZj1dAx4q7vvzqPdaSpNgBARkd6UZYhJRER6pAAhIiKJFCBERCSRAoSIiCRSgBARkUQKEFJKZtYys7tiXydc2Wxm7zCzK5+Bv/uIma1bxPN+zsw+bGZrzUxH8komCncmtUhGjrv7toU+2N3/PM3GLMDLCBafvRz4l5zbIiWhACESE27zcSPwivDSL7v7g2b2YWDM3T9uZu8B3kGwMOwed3+zmZ0GXEewMd8xYLu77zGz04GdwBDw78T28DGzXyHYKruPYJHjb7l7q6s9lwMfCH/vZcAG4IiZ/ZS7vz6N/wYiEQ0xSVkNdg0xXR6774i7X0SwUvaPE557NfBCd38BQaAA+AhwZ3jtg8DnwusfAv453KTvJoIVuZjZBQSrj18SZjIt4C3df8jdbyTYt+lud/8x4O7wbys4SOqUQUhZnWiIaWfs+6cS7t8DfN7M/o5gKwaAlwL/GcDdv25mp5vZaoIhoTeG1//BzJ4KH38J8BPAd4JdGxhk7s36zgUeCm8vC88AEUmdAoTIbD7H7cjPE3T8rwf+h5k9jxNv/5z0Owy4wd0/cKKGmNlugr2BamZ2D7DRzO4C3u3u3zrxyxA5ORpiEpnt8tj3f4vfYWYVYLO7f4PgYKM1wArgNsIhonCjwQPhGR3x65cC0bnTtwK/aGbrw/tOM7NndTfE3YeBfyCoP3wM+D1336bgIFlQBiFlNRh+Eo981d2jqa79ZnY7wQeoK7qeVwX+Khw+MoJzl58Oi9h/aWZ7CIrU0RbRHwF2mtl3gW8S7AiKu99jZv8d+FoYdBrAO4FHE9r6IoJi9m8Bn0y4XyQV2s1VJCacxTTs7gfybotI3jTEJCIiiZRBiIhIImUQIiKSSAFCREQSKUCIiEgiBQgREUmkACEiIokUIEREJNH/B/UerFph3iEUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f5c7e3fbac8>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(1, len(scores)+1), scores)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episode #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Watch a Smart Agent!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAD8CAYAAAB9y7/cAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAABCFJREFUeJzt28ttwlAQQFE7chOpI22kDqgp1JE2qCNlOBs2SVhE4vPM5RyJBUigWdhXI+sxr+s6AdDzMnoAAG5D4AGiBB4gSuABogQeIErgAaIEHiBK4AGiBB4gahk9wIm/0wL8NV/yZRs8QJTAA0QJPECUwANECTxAlMADRAk8QJTAA0QJPECUwANECTxAlMADRAk8QJTAA0QJPECUwANECTxAlMADRAk8QJTAA0QJPECUwANECTxAlMADRAk8QJTAA0QJPECUwANECTxAlMADRAk8QJTAA0QJPECUwANECTxAlMADRAk8QJTAA0QJPECUwANECTxAlMADRAk8QJTAA0QJPECUwANECTxAlMADRAk8QJTAA0QJPECUwANECTxAlMADRAk8QJTAA0QJPECUwANECTxAlMADRAk8QJTAA0QJPECUwANECTxAlMADRAk8QJTAA0QJPECUwANECTxAlMADRAk8QJTAA0QJPECUwANECTxAlMADRAk8QJTAA0QJPECUwANECTxAlMADRAk8QJTAA0QJPECUwANECTxAlMADRAk8QJTAA0QJPEDUMnoAGOl42J/9/G33cedJ4Pps8ABRAg8QJfAAUQIPECXwAFECz1NzWoYygQeIEniAKIEHiBJ4gCiBB4gSeIAogQeIEniAKIEHiBJ4gCiBB4gSeIAogQeIEniAKIEHiBJ4gCiBB4gSeIAogQeIEniAKIEHiBJ4gCiBB4gSeIAogQeIEniAKIEHiBJ4gCiBB4gSeIAogQeIEniAKIEHiBJ4gCiBB4gSeIAogYczjof96BHgYgIPECXwAFECDxAl8ABRAg8QJfAAUQIPECXwAFECDxAl8ABRAg8QJfAAUQIPECXwAFECDxAl8ABRAg8QJfAkzfP879ctfwNGEniAqGX0ALAFn1+7H+/fXw+DJoHrscHDGb+DD49I4Hl6Yk6VwPP0PI6hSuDhDNGnYF7XdfQM0zRNmxiCjnsfXdzIfUTPRRfyJk7ROEfMo3MNcwuXLg6bCLzth2uzwYNn8ABZAg8QJfAAUQIPECXwAFECDxAl8ABRmzgHD9fmXDrY4AGyBB4gSuABogQeIErgAaIEHiBK4AGiBB4gSuABogQeIErgAaIEHiBK4AGiBB4gSuABogQeIErgAaIEHiBK4AGiBB4gSuABogQeIErgAaIEHiBK4AGiBB4gSuABogQeIErgAaIEHiBK4AGiBB4gSuABogQeIErgAaIEHiBK4AGiBB4gSuABogQeIErgAaIEHiBK4AGiBB4gSuABopbRA5zMowcAqLHBA0QJPECUwANECTxAlMADRAk8QJTAA0QJPECUwANECTxAlMADRAk8QJTAA0QJPECUwANECTxAlMADRAk8QJTAA0QJPECUwANECTxAlMADRH0DVPIoPyY64ooAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f5c78e7db70>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "state = env.reset()\n",
    "img = plt.imshow(env.render(mode='rgb_array'))\n",
    "for t in range(200):\n",
    "    action = policy.act(state)\n",
    "    img.set_data(env.render(mode='rgb_array')) \n",
    "    plt.axis('off')\n",
    "    display.display(plt.gcf())\n",
    "    display.clear_output(wait=True)\n",
    "    state, reward, done, _ = env.step(action)\n",
    "    if done:\n",
    "        break \n",
    "\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}