temporal_difference.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "# Temporal Difference Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import copy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## Bellman's Equation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "$$\n",
    "v_\\pi (s) = \\sum_a \\pi (a | s) \\left[ \\sum_{s'} p (s' | s,a) \\left[ r(s',s,a) + \\gamma v_\\pi (s') \\right]  \\right]\n",
    "$$\n",
    "$$\n",
    "\\vec{v} = T \\left( \\vec{\\pi} \\odot \\left[ (P \\odot R) \\vec{1} + \\gamma P \\vec{v} \\right] \\right)\n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "# MDP Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "class MDP():\n",
    "    \n",
    "    def __init__(self, P, R, terminal=[], Pi=None, gamma=0.9):\n",
    "        '''\n",
    "         MDP(P, R, Pi=None, gamma=0.9)\n",
    "         \n",
    "         P  is an n_states x n_actions x n_states array\n",
    "         R  is an n_states x n_actions array\n",
    "         terminal is a list of terminal states\n",
    "         Pi is an n_states x n_actions array\n",
    "         gamma is the temporal discount factor\n",
    "        '''\n",
    "        self.n_states, self.n_actions, _ = np.shape(P)\n",
    "        self.P = P                      # State transition function\n",
    "        self.R = R                      # Reward function\n",
    "        self.terminal = terminal        # Terminal states\n",
    "        self.gamma = gamma              # Temporal discount factor\n",
    "        if Pi!=None:\n",
    "            self.Pi = Pi                # Policy function\n",
    "        else:\n",
    "            self.Pi = np.ones((self.n_states, self.n_actions)) / self.n_actions\n",
    "        \n",
    "        self.v = np.zeros(self.n_states)                    # Value function\n",
    "        self.Q = np.zeros((self.n_states, self.n_actions))  # Q function\n",
    "        self.E = np.zeros((self.n_states, self.n_actions))  # Elegibility function\n",
    "        \n",
    "        self.M = np.zeros((self.n_states, self.n_actions, 3)) # State transition model (for model-based RL)\n",
    "        # M[s,a,0] is the anticipated reward\n",
    "        # M[s,a,1] is the anticipated new state\n",
    "        # M[s,a,2] is 1 if (s,a) was experienced, otherwise 0\n",
    "        \n",
    "        self.b = []  # Used for full policy evaluation\n",
    "        self.B = []  # \"\n",
    "        self.state = 0       # Current state\n",
    "\n",
    "    def AddTerminalState(self, t):\n",
    "        self.terminal.append(t)\n",
    "        \n",
    "    def Step(self, action=None):\n",
    "        '''\n",
    "         mdp.Step(action=None)\n",
    "         \n",
    "         Take a step by choosing an action.\n",
    "         \n",
    "         Inputs:\n",
    "           action is the index of the action to choose.\n",
    "                  If None, then it chooses an action according to the policy.\n",
    "        '''\n",
    "        if action is None:\n",
    "            action = self.SampleAction()\n",
    "        \n",
    "        # New state\n",
    "        newstate_idx = self.sample(self.P[self.state, action, :])\n",
    "        reward = self.R[self.state, action]\n",
    "        \n",
    "        # Is this a terminal state?\n",
    "        done = 0\n",
    "        if newstate_idx in(self.terminal):\n",
    "            #if np.max(self.P[newstate_idx, :, newstate_idx]) == 1.:\n",
    "            done = 1\n",
    "        return self.state, action, reward, newstate_idx, done\n",
    "    \n",
    "    def sample(self, p):\n",
    "        '''\n",
    "         idx = mdp.sample(p)\n",
    "         \n",
    "         Randomly choose an index according to the probability vector p.\n",
    "         \n",
    "         Input:\n",
    "           p  is a probability vector\n",
    "           \n",
    "         Output:\n",
    "           idx is an index of the vector p\n",
    "        '''\n",
    "        c = copy.deepcopy(p)\n",
    "        for k in range(1,len(c)):\n",
    "            c[k] += c[k-1]\n",
    "        r = np.random.rand()\n",
    "        result = np.where(r>c)\n",
    "        return len(result[0])\n",
    "    \n",
    "    def SampleAction(self, state=None, Pi=None, eps=0.):\n",
    "        '''\n",
    "         a = mdp.SampleAction(state=None)\n",
    "         \n",
    "         Choose an action.\n",
    "         \n",
    "         Input:\n",
    "           state is the state to choose an action from.\n",
    "                 If None, then use mdp.state.\n",
    "           Pi is the policy. If None, then use mdp.Pi.\n",
    "           eps is the epsilon to use for epsilon-greedy action selection.\n",
    "        '''\n",
    "        if state is None:\n",
    "            state = self.state\n",
    "        if Pi is None:\n",
    "            Pi = self.Pi\n",
    "        Pi_eps = copy.deepcopy(Pi)\n",
    "        Pi_eps += eps\n",
    "        for pi_row in Pi_eps:\n",
    "            pi_row /= np.sum(pi_row)\n",
    "        return self.sample(Pi_eps[state,:])\n",
    "        \n",
    "    def LinearSystem(self, Pi=None, R=None, P=None, gamma=None):\n",
    "        list_like = (np.ndarray,)\n",
    "        if not isinstance(Pi, list_like):\n",
    "            Pi = self.Pi\n",
    "        if not isinstance(P, list_like):\n",
    "            P = self.P\n",
    "        if not isinstance(R, list_like):\n",
    "            R = self.R\n",
    "        if gamma==None:\n",
    "            gamma = self.gamma\n",
    "        self.b = np.sum( Pi * R, axis=1 )\n",
    "        self.B = gamma * np.sum(Pi[:,:,np.newaxis]*P, axis=1)\n",
    "\n",
    "    def EvaluatePolicy(self):\n",
    "        self.LinearSystem()\n",
    "        #B = self.gamma * np.einsum('ijk,ijk->ik', self.Pi[:,:,np.newaxis], self.P)\n",
    "        v = np.linalg.solve(np.eye(self.n_states)-self.B, self.b)\n",
    "        self.v = v\n",
    "        return v\n",
    "\n",
    "    def EvaluateStep(self, n_steps=100, tol=1.e-4):\n",
    "        '''\n",
    "         EvaluateStep(n_steps=100, tol=1.e-4)\n",
    "         \n",
    "         Iteratively maps the current value vector using the Bellman equation.\n",
    "         The iterations stop when\n",
    "           - it reaches n_steps, or\n",
    "           - if max(abs(v-vnew))<tol,\n",
    "         whichever comes first.\n",
    "        '''\n",
    "        self.LinearSystem()\n",
    "        delta_V = 1.e50\n",
    "        k = 0\n",
    "        while delta_V>=tol and k<n_steps:\n",
    "            v_copy = copy.deepcopy(self.v)\n",
    "            self.v = self.B @ v_copy + self.b\n",
    "            k += 1\n",
    "            delta_V = np.max(np.abs(self.v-v_copy))\n",
    "        #print(str(k)+' steps')\n",
    "        return self.v\n",
    "\n",
    "    def ValueIteration(self, n_steps=100, tol=1.e-4):\n",
    "        '''\n",
    "         Pi = mdp.ValueIteration(n_steps=100, tol=1.e-4)\n",
    "         \n",
    "         Performs value iteration, starting from the current value vector.\n",
    "         The iterations stop when\n",
    "           - it reaches n_steps, or\n",
    "           - if max(abs(v-vnew))<tol,\n",
    "         whichever comes first.\n",
    "        '''\n",
    "        delta_V = 1.e50\n",
    "        k = 0\n",
    "        while delta_V>=tol and k<n_steps:\n",
    "            v_copy = copy.deepcopy(self.v)\n",
    "            self.LinearSystem()\n",
    "            v_new = np.zeros_like(self.R)\n",
    "            for a in range(n_actions):\n",
    "                v_new[:,a] = self.B@self.v + self.b\n",
    "            self.v = np.max(v_new, axis=1)\n",
    "            delta_V = np.max(np.abs(self.v - v_copy))\n",
    "            k += 1\n",
    "        \n",
    "        return self.OptimalPolicy()\n",
    "        \n",
    "    def PolicyIteration(self):\n",
    "        '''\n",
    "         Pi = mdp.PolicyIteration()\n",
    "         \n",
    "         Performs policy iteration, starting from the current value vector.\n",
    "         The iteration stops when the policy does not change in an iteration.\n",
    "        '''\n",
    "        delta_Pi = 1.\n",
    "        Pi_copy = copy.deepcopy(self.Pi)\n",
    "        while delta_Pi!=0:\n",
    "            self.EvaluateStep(n_steps=1)\n",
    "            Pi_new = self.OptimalPolicy()\n",
    "            delta_Pi = np.max(np.abs(Pi_copy - Pi_new))\n",
    "            Pi_copy = copy.deepcopy(Pi_new)\n",
    "            \n",
    "        return Pi_new\n",
    "        \n",
    "    def OptimalPolicyFromQ(self):\n",
    "        '''\n",
    "         Pi = mdp.OptimalPolicyFromQ()\n",
    "         \n",
    "         Returns the optimal policy from the current Q function.\n",
    "        '''\n",
    "        Pi_opt = np.zeros_like(self.Pi)\n",
    "        for s in range(self.n_states):\n",
    "            q_star = np.max(self.Q[s,:])\n",
    "            blah = self.Q[s,:]==q_star\n",
    "            Pi_opt[s,:] = blah / np.sum(blah)\n",
    "            \n",
    "        return Pi_opt\n",
    "        \n",
    "    def OptimalPolicy(self):\n",
    "        '''\n",
    "         Pi = mdp.OptimalPolicy()\n",
    "\n",
    "         Returns the optimal policy for the current value function.\n",
    "        '''\n",
    "        if len(self.v)==0:\n",
    "            self.v = np.zeros(self.n_states)\n",
    "\n",
    "        v_new = np.zeros((self.n_states,self.n_actions), dtype=float)\n",
    "        for a in range(self.n_actions):\n",
    "            Pi_a = np.zeros_like(self.Pi)\n",
    "            Pi_a[:,a] = 1.\n",
    "\n",
    "            self.LinearSystem(Pi=Pi_a)\n",
    "            v_new[:,a] = self.B@self.v + self.b\n",
    "\n",
    "        Pi_opt = np.zeros_like(self.Pi)\n",
    "        for s in range(self.n_states):\n",
    "            a_star = np.max(v_new[s,:])\n",
    "            blah = v_new[s,:]==a_star\n",
    "            Pi_opt[s,:] = blah / np.sum(blah)\n",
    "            \n",
    "        return Pi_opt\n",
    "\n",
    "            \n",
    "    def Episode(self):\n",
    "        '''\n",
    "         trajectory = mdp.Episode()\n",
    "         \n",
    "         Creates a trajectory that starts at the current state and follows the optimal policy.\n",
    "         \n",
    "         Output:\n",
    "           trajectory is a list of states\n",
    "        '''\n",
    "        max_iters = 100\n",
    "        done = 0\n",
    "        n_iters = 0\n",
    "        s = self.state\n",
    "        traj = [s]\n",
    "        while not done and n_iters<max_iters:\n",
    "            oldstate, action, reward, newstate, done = self.Step()\n",
    "            traj.append(newstate)\n",
    "            self.state = newstate\n",
    "            n_iters += 1\n",
    "            \n",
    "        return traj\n",
    "    \n",
    "    \n",
    "    def SARSA_lambda(self, alpha=0.1, eps=0.1, lam=0.9, trace='dutch'):\n",
    "        '''\n",
    "         mdp.SARSA_lambda(alpha=0.1, eps=0.1, lam=0.9, trace='dutch')\n",
    "         \n",
    "         Performs an episode using SARSA(lambda).\n",
    "         The method uses eligibility traces with decay lam.\n",
    "         It uses the current state, and updates mdp.Q and mdp.E.\n",
    "         \n",
    "         Inputs:\n",
    "           alpha  TD learning rate\n",
    "           eps    epsilon used for epsilon-greedy action selection\n",
    "           lam    trace decay\n",
    "           trace  type of eligibility increment, one of:\n",
    "                   'dutch', 'accumulating', or 'replacing'\n",
    "        '''\n",
    "        iters_max = 100\n",
    "        # Current state is in self.state\n",
    "        # Choose an action using policy\n",
    "        A = self.SampleAction(Pi=self.OptimalPolicyFromQ(), eps=eps)\n",
    "        # for each step of an episode\n",
    "        done = 0\n",
    "        n_iters = 0\n",
    "        traj = [self.state]\n",
    "        self.E[:,:] = 0.\n",
    "        while not done and n_iters<iters_max:\n",
    "            # Take action, get reward and newstate\n",
    "            oldstate, action, reward, newstate, done = self.Step(action=A)\n",
    "            # Choose newaction from newstate\n",
    "            newaction = self.SampleAction(state=newstate, Pi=self.OptimalPolicyFromQ(), eps=eps)\n",
    "            \n",
    "            # Compute the return delta\n",
    "            delta = reward + self.gamma*self.Q[newstate,newaction] - self.Q[self.state,A]\n",
    "            \n",
    "            # Increment self.E for current state/action\n",
    "            if trace=='dutch':\n",
    "                self.E[self.state,A] += 1. - alpha*self.E[self.state,A]   # Dutch trace\n",
    "            elif trace=='accumulating':\n",
    "                self.E[self.state,A] += 1.                                # Accumulating trace\n",
    "            else:\n",
    "                self.E[self.state,A] = 1.                                 # Replacing trace\n",
    "            \n",
    "            for s in range(self.n_states):\n",
    "                for a in range(self.n_actions):\n",
    "                    # Update self.Q\n",
    "                    self.Q[s,a] += alpha*delta*self.E[s,a]\n",
    "                    \n",
    "                    # Update self.E\n",
    "                    self.E[s,a] *= self.gamma*lam\n",
    "                    \n",
    "            # Switch state to newstate, and action to newaction\n",
    "            self.state = newstate\n",
    "            traj.append(newstate)\n",
    "            A = newaction\n",
    "            n_iters += 1\n",
    "        \n",
    "        return traj\n",
    "        \n",
    "        \n",
    "    def SARSA(self, alpha=0.1, eps=0.1):\n",
    "        '''\n",
    "         mdp.SARSA(alpha=0.1, eps=0.1)\n",
    "         \n",
    "         Performs an episode using SARSA, updating mdp.Q as it goes.\n",
    "         It uses the current state.\n",
    "        '''\n",
    "        iters_max = 100\n",
    "        # Current state is in self.state\n",
    "        # Choose an action using policy\n",
    "        a = self.SampleAction(Pi=self.OptimalPolicyFromQ(), eps=eps)\n",
    "        # for each step of an episode\n",
    "        done = 0\n",
    "        n_iters = 0\n",
    "        while not done and n_iters<iters_max:\n",
    "            # Take action, get reward and newstate\n",
    "            oldstate, action, reward, newstate, done = self.Step(action=a)\n",
    "            # Choose newaction from newstate\n",
    "            newaction = self.SampleAction(state=newstate, Pi=self.OptimalPolicyFromQ(), eps=eps)\n",
    "            \n",
    "            # Update self.Q\n",
    "            self.Q[self.state, a] = (1-alpha)*self.Q[self.state,a] + alpha*( reward + self.gamma*self.Q[newstate,newaction] )\n",
    "            # Switch state to newstate, and action to newaction\n",
    "            self.state = newstate\n",
    "            a = newaction\n",
    "            n_iters += 1\n",
    "\n",
    "            \n",
    "    def DynaQ(self, alpha=0.1, eps=0.1, n=5):\n",
    "        '''\n",
    "         traj = mdp.DynaQ(alpha=0.1, eps=0.1, n=5)\n",
    "         \n",
    "         Performs an episode of Dyna-Q model-based Q-learning.\n",
    "         Starting from the current state, it takes n planning steps for each\n",
    "         actual step. It updates mdp.Q and mdp.M, and returns the trajectory.\n",
    "        '''\n",
    "        done = 0\n",
    "        iters_max = 100\n",
    "        n_iters = 0\n",
    "        traj = [self.state]\n",
    "        while not done and n_iters<iters_max:\n",
    "            # Choose an action using policy\n",
    "            A = self.SampleAction(Pi=self.OptimalPolicyFromQ(), eps=eps)\n",
    "            \n",
    "            # Take action, get reward and newstate\n",
    "            oldstate, action, reward, newstate, done = self.Step(action=A)\n",
    "            \n",
    "            # Update self.Q\n",
    "            maxQ = np.max(self.Q[newstate,:])\n",
    "            self.Q[self.state,A] = (1-alpha)*self.Q[self.state,A] + alpha*( reward + self.gamma*maxQ )\n",
    "            \n",
    "            traj.append(newstate)\n",
    "            \n",
    "            # n planning steps\n",
    "            for nn in range(n):\n",
    "                s,a = self.ChoosePreviousExperience()\n",
    "                if s is not None:\n",
    "                    R = self.M[s,a,0]\n",
    "                    planningstate = self.M[s,a,1]\n",
    "                    maxQ = np.max(self.Q[planningstate,:])\n",
    "                    self.Q[s,a] += alpha * ( R + self.gamma*maxQ - self.Q[s,a] )\n",
    "            \n",
    "            # Switch state to newstate\n",
    "            self.state = newstate\n",
    "            n_iters += 1\n",
    "            \n",
    "    def ChoosePreviousExperience(self):\n",
    "        '''\n",
    "         s,a = mdp.ChoosePreviousExperience()\n",
    "         \n",
    "         Returns a state/action pair that has been done before, based on the\n",
    "         model stored in mdp.M.\n",
    "        '''\n",
    "        r,c = np.nonzero(self.M[:,:,2])\n",
    "        coords = list(zip(r,c))\n",
    "        if len(coords)>0:\n",
    "            idx = np.random.randint(0,len(coords))\n",
    "            return coords[idx]\n",
    "        else:\n",
    "            return None, None\n",
    "            \n",
    "    def QLearning(self, alpha=0.1, eps=0.1):\n",
    "        iters_max = 100\n",
    "        # Current state is in self.state\n",
    "        # for each step of an episode\n",
    "        done = 0\n",
    "        n_iters = 0\n",
    "        while not done and n_iters<iters_max:\n",
    "            \n",
    "            # Choose an action using policy\n",
    "            a = self.SampleAction(Pi=self.OptimalPolicyFromQ(), eps=eps)\n",
    "            \n",
    "            # Take action, get reward and newstate\n",
    "            oldstate, action, reward, newstate, done = self.Step(action=a)\n",
    "            \n",
    "            # Update self.Q\n",
    "            maxQ = np.max(self.Q[newstate,:])\n",
    "            self.Q[self.state, a] = (1-alpha)*self.Q[self.state,a] + alpha*( reward + self.gamma*maxQ )\n",
    "            \n",
    "            # Switch state to newstate, and action to newaction\n",
    "            self.state = newstate\n",
    "            n_iters += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "M = np.zeros((5,5,2), dtype=np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "M[2,2,1] = 1.\n",
    "M[1,2,1] = 1.\n",
    "M[3,1,1] = 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(1, 2), (2, 2), (3, 1)]\n",
      "(2, 2)\n"
     ]
    }
   ],
   "source": [
    "r,c = np.nonzero(M[:,:,1])\n",
    "coords = list(zip(r,c))\n",
    "print(coords)\n",
    "idx = np.random.randint(len(coords))\n",
    "print(coords[idx])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 2)\n",
      "(2, 2)\n",
      "(3, 1)\n"
     ]
    }
   ],
   "source": [
    "for coord in zip(r,c):\n",
    "    print(coord)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "true"
   },
   "source": [
    "## 1D GridWorld"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Create arrays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "gs = 8  # grid length\n",
    "n_actions = 2  # number of actions\n",
    "golden_square_idx = 4\n",
    "golden_square_reward = 4.\n",
    "terminal = [golden_square_idx]\n",
    "movement_cost = -0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.5 0.5]\n",
      " [0.5 0.5]\n",
      " [0.5 0.5]\n",
      " [0.5 0.5]\n",
      " [0.5 0.5]\n",
      " [0.5 0.5]\n",
      " [0.5 0.5]\n",
      " [0.5 0.5]]\n"
     ]
    }
   ],
   "source": [
    "Pi = np.ones((gs,n_actions)) / n_actions\n",
    "print(Pi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[[1. 0. 0. 0. 0. 0. 0. 0.]\n",
      "  [0. 1. 0. 0. 0. 0. 0. 0.]]\n",
      "\n",
      " [[1. 0. 0. 0. 0. 0. 0. 0.]\n",
      "  [0. 0. 1. 0. 0. 0. 0. 0.]]\n",
      "\n",
      " [[0. 1. 0. 0. 0. 0. 0. 0.]\n",
      "  [0. 0. 0. 1. 0. 0. 0. 0.]]\n",
      "\n",
      " [[0. 0. 1. 0. 0. 0. 0. 0.]\n",
      "  [0. 0. 0. 0. 1. 0. 0. 0.]]\n",
      "\n",
      " [[1. 0. 0. 0. 0. 0. 0. 0.]\n",
      "  [0. 0. 0. 0. 0. 0. 0. 1.]]\n",
      "\n",
      " [[0. 0. 0. 0. 1. 0. 0. 0.]\n",
      "  [0. 0. 0. 0. 0. 0. 1. 0.]]\n",
      "\n",
      " [[0. 0. 0. 0. 0. 1. 0. 0.]\n",
      "  [0. 0. 0. 0. 0. 0. 0. 1.]]\n",
      "\n",
      " [[0. 0. 0. 0. 0. 0. 1. 0.]\n",
      "  [0. 0. 0. 0. 0. 0. 0. 1.]]]\n"
     ]
    }
   ],
   "source": [
    "# State Transition Probabilities: 1D Grid\n",
    "# P[s,a,t] is the probability of action a taking you from state s to state t\n",
    "# P[s,a,t] = p(t|a,s)\n",
    "P = np.zeros((gs,n_actions,gs))\n",
    "P[0,0,0] = 1.  # Given: action 'decrease' and state 0, prob of ending in state 0\n",
    "P[0,1,1] = 1.\n",
    "for l in range(1,gs-1):\n",
    "    if l==golden_square_idx:\n",
    "        # 'decrease' takes you to 0, 'increase' takes you to last\n",
    "        P[l,0,0] = 1.\n",
    "        P[l,1,-1] = 1.\n",
    "    else:\n",
    "        P[l,0,l-1] = 1.\n",
    "        P[l,1,l+1] = 1.\n",
    "P[-1,1,-1] = 1.\n",
    "P[-1,0,-2] = 1.\n",
    "\n",
    "print(P)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-2.  -0.5]\n",
      " [-0.5 -0.5]\n",
      " [-0.5 -0.5]\n",
      " [-0.5  4. ]\n",
      " [ 0.   0. ]\n",
      " [ 4.  -0.5]\n",
      " [-0.5 -0.5]\n",
      " [-0.5 -2. ]]\n"
     ]
    }
   ],
   "source": [
    "# Reward Function: 1D Grid\n",
    "# R[s,a] is the reward of taking action a from state s to state t\n",
    "# R[s,a] = r(s,a)\n",
    "R = np.zeros((gs,n_actions))\n",
    "R[0,0] = -2.   # Penalty for trying to go off the grid\n",
    "R[0,1] = movement_cost\n",
    "R[-1,1] = -2.  # Penalty for trying to go off the grid\n",
    "R[-1,0] = movement_cost\n",
    "for l in range(1,gs-1):\n",
    "    # Reward (cost) of a normal movement\n",
    "    R[l,0] = movement_cost\n",
    "    R[l,1] = movement_cost\n",
    "R[golden_square_idx-1,1] = golden_square_reward\n",
    "R[golden_square_idx+1,0] = golden_square_reward\n",
    "R[golden_square_idx,:] = 0.\n",
    "\n",
    "print(R)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Create Markov Decision Process object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "gw = MDP(P, R, terminal=terminal)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "true"
   },
   "source": [
    "### Find optimal policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "old state: 0\n",
      "   action: 0\n",
      "   reward: -2.0\n",
      "new state: 0\n",
      "     done: 0\n"
     ]
    }
   ],
   "source": [
    "oldstate, action, reward, newstate, done = gw.Step()\n",
    "print('old state: '+str(oldstate))\n",
    "print('   action: '+str(action))\n",
    "print('   reward: '+str(reward))\n",
    "print('new state: '+str(newstate))\n",
    "print('     done: '+str(done))\n",
    "gw.state = newstate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 1.],\n",
       "       [0., 1.],\n",
       "       [0., 1.],\n",
       "       [0., 1.],\n",
       "       [0., 1.],\n",
       "       [1., 0.],\n",
       "       [1., 0.],\n",
       "       [1., 0.]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gw.ValueIteration()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-6.60280705, -5.29231973, -4.04679234, -2.58944104, -5.59640996,\n",
       "       -2.72689723, -4.35225054, -5.83365954])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gw.EvaluatePolicy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "gw.Pi = gw.PolicyIteration()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5.49302123, 6.65891247, 7.95434719, 9.3937191 , 5.99302123,\n",
       "       9.3937191 , 7.95434719, 6.65891247])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gw.EvaluatePolicy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "true"
   },
   "source": [
    "### SARSA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "for n_episodes in range(500):\n",
    "    gw.state = np.random.randint(0,gw.n_states)\n",
    "    #gw.state = np.random.choice([0,gw.n_states-1])\n",
    "    gw.SARSA(alpha=0.1, eps=0.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-1.37361181  1.26902046]\n",
      " [-0.01096624  2.81836005]\n",
      " [ 1.26559583  3.50690555]\n",
      " [ 2.2033877   5.01520762]\n",
      " [ 0.23777703  1.52556548]\n",
      " [ 5.09877781  2.027761  ]\n",
      " [ 3.61268523  1.30850712]\n",
      " [ 2.65205939 -0.51865569]]\n"
     ]
    }
   ],
   "source": [
    "print(gw.Q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 1.]\n",
      " [0. 1.]\n",
      " [0. 1.]\n",
      " [0. 1.]\n",
      " [0. 1.]\n",
      " [1. 0.]\n",
      " [1. 0.]\n",
      " [1. 0.]]\n"
     ]
    }
   ],
   "source": [
    "gw.Pi = gw.OptimalPolicyFromQ()\n",
    "print(gw.Pi)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "true"
   },
   "source": [
    "### Q-Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "for n_episodes in range(50):\n",
    "    gw.state = np.random.randint(0,gw.n_states)\n",
    "    #gw.state = np.random.choice([0,gw.n_states-1])\n",
    "    gw.QLearning(alpha=0.1, eps=0.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 1.]\n",
      " [0. 1.]\n",
      " [0. 1.]\n",
      " [0. 1.]\n",
      " [0. 1.]\n",
      " [1. 0.]\n",
      " [1. 0.]\n",
      " [1. 0.]]\n"
     ]
    }
   ],
   "source": [
    "gw.Pi = gw.OptimalPolicyFromQ()\n",
    "print(gw.Pi)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## 2D Gridworld"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Define the 2D MDP class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 513,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "class MDP_2d(MDP):\n",
    "    def __init__(self, P, R, side=5, terminal=[], Pi=None, gamma=0.9):\n",
    "        '''\n",
    "         MDP(P, R, terminal=[], Pi=None, gamma=0.9)\n",
    "         \n",
    "         P  is an n_states x n_actions x n_states array\n",
    "         R  is an n_states x n_actions array\n",
    "         side is the length of the side of the arena\n",
    "         terminal is a list of coords for terminal states\n",
    "         Pi is an n_states x n_actions array\n",
    "        '''\n",
    "        terminal_idx = []\n",
    "        for t in terminal:\n",
    "            terminal_idx.append(coord2idx(t, side))\n",
    "        MDP.__init__(self, P,R,terminal=terminal_idx,Pi=Pi,gamma=gamma)\n",
    "        self.side = side\n",
    "        \n",
    "    def ShowValue(self):\n",
    "        print( np.round( np.reshape(self.v, (self.side,self.side)), 1) )\n",
    "    \n",
    "    def ShowPolicy(self):\n",
    "        for a in range(self.n_actions):\n",
    "            print('Action '+str(a))\n",
    "            print(np.reshape(self.Pi[:,a], (self.side,self.side)))\n",
    "\n",
    "    def ShowFunction(self, M):\n",
    "        '''\n",
    "         mdp2.ShowFunction(M)\n",
    "         \n",
    "         Displays the (state,action) function M, showing the spatial 2D M for\n",
    "         for one action at a time.\n",
    "        '''\n",
    "        for a in range(self.n_actions):\n",
    "            print('Action '+str(a))\n",
    "            print(np.round(np.reshape(M[:,a], (self.side,self.side)), 3))\n",
    "        \n",
    "    def ShowQ(self):\n",
    "        for a in range(self.n_actions):\n",
    "            print('Action '+str(a))\n",
    "            print(np.round(np.reshape(self.Q[:,a], (self.side,self.side)), 3))\n",
    "            \n",
    "    def ShowR(self):\n",
    "        for a in range(self.n_actions):\n",
    "            print('Action '+str(a))\n",
    "            print(np.round(np.reshape(self.R[:,a], (self.side,self.side)), 3))\n",
    "        \n",
    "    def PlotTrajectory(self, traj, fig=None):\n",
    "        tcoords = []\n",
    "        for t in traj:\n",
    "            tcoords.append(idx2coord(t, self.side))\n",
    "        tcoords = np.array(tcoords)\n",
    "        if fig is None:\n",
    "            fig = plt.figure(figsize=(8,8))\n",
    "        plt.axis([-1,self.side,-1,self.side])\n",
    "        plt.plot(tcoords[:,1], tcoords[:,0], 'o--')\n",
    "        plt.grid(True)\n",
    "        return fig"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "true"
   },
   "source": [
    "### Set up the matrices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "gl = 5  # grid length\n",
    "gs = gl*gl\n",
    "n_actions = 4  # number of actions\n",
    "A_coords = (0,1)\n",
    "A_reward = 10.\n",
    "Ap_coords = (4,1)\n",
    "B_coords = (0,3)\n",
    "B_reward = 5\n",
    "Bp_coords = (2,3)\n",
    "C_coords = (2,2)\n",
    "C_reward = 2\n",
    "terminal = [A_coords, B_coords]\n",
    "movement_cost = 0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "def coord2idx(coords, row_length):\n",
    "    return coords[0]*row_length + coords[1]\n",
    "def idx2coord(idx, row_length):\n",
    "    row = idx//row_length\n",
    "    col = idx%row_length\n",
    "    return (row,col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# State Transition Probabilities: 2D Grid\n",
    "# P[s,a,t] is the probability of action a taking you from state s to state t\n",
    "# P[s,a,t] = p(t|a,s)\n",
    "# 0==up, 1==right, 2==down, 3==left\n",
    "P = np.zeros((gs,n_actions,gs))\n",
    "for sr in range(gl):\n",
    "    sr_offset = sr*gl\n",
    "    for sc in range(gl):\n",
    "        #idx = sr_offset + sc\n",
    "        idx = coord2idx([sr,sc], gl)\n",
    "        for a in range(4):\n",
    "            img = np.zeros((gl,gl))\n",
    "            if sr==A_coords[0] and sc==A_coords[1]:   # A, any action\n",
    "                img[Ap_coords[0],Ap_coords[1]] = 1.\n",
    "            elif sr==B_coords[0] and sc==B_coords[1]: # B, any action\n",
    "                img[Bp_coords[0],Bp_coords[1]] = 1.\n",
    "            elif sr==0 and a==0:    # top row, up\n",
    "                img[sr,sc] = 1.\n",
    "            elif sr==gl-1 and a==2: # bottom row, down\n",
    "                img[sr,sc] = 1.\n",
    "            elif sc==0 and a==3:    # left col, left\n",
    "                img[sr,sc] = 1.\n",
    "            elif sc==gl-1 and a==1: # right col, right\n",
    "                img[sr,sc] = 1.\n",
    "            elif a==0:              # up\n",
    "                img[sr-1,sc] = 1.\n",
    "            elif a==1:              # right\n",
    "                img[sr,sc+1] = 1.\n",
    "            elif a==2:              # down\n",
    "                img[sr+1,sc] = 1.\n",
    "            elif a==3:              # left\n",
    "                img[sr,sc-1] = 1.\n",
    "            P[idx,a,:] = img.flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Reward Function: 2D Grid\n",
    "# R[s,a] is the reward of taking action a from state s to state t\n",
    "# R[s,a] = r(s,a)\n",
    "R = np.zeros((gs,n_actions))\n",
    "for sr in range(gl):\n",
    "    sr_offset = sr*gl\n",
    "    for sc in range(gl):\n",
    "        #idx = sr_offset + sc\n",
    "        idx = coord2idx([sr,sc], gl)\n",
    "        #svec = np.zeros(gs)\n",
    "        #svec[idx] = 1.\n",
    "        for a in range(4):\n",
    "            reward = 0.\n",
    "            tvec = P[idx,a,:]\n",
    "            tidx = np.argmax(tvec)\n",
    "            tcoords = idx2coord(tidx, gl)\n",
    "            if tcoords==A_coords:\n",
    "                reward += A_reward\n",
    "            elif tcoords==B_coords:\n",
    "                reward += B_reward\n",
    "            elif tcoords==C_coords:\n",
    "                reward += C_reward\n",
    "            elif sr==0 and a==0:    # top row, up\n",
    "                reward += -1.\n",
    "            elif sr==gl-1 and a==2: # bottom row, down\n",
    "                reward += -1.\n",
    "            elif sc==0 and a==3:    # left col, left\n",
    "                reward += -1.\n",
    "            elif sc==gl-1 and a==1: # right col, right\n",
    "                reward += -1.\n",
    "            R[idx,a] = reward"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "true"
   },
   "source": [
    "### Create MDP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "grid2d = MDP_2d(P, R, side=5, terminal=terminal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Action 0\n",
      "[[-1. -1. -1. -1. -1.]\n",
      " [ 0. 10.  0.  5.  0.]\n",
      " [ 0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  2.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.]]\n",
      "Action 1\n",
      "[[10.  0.  5.  0. -1.]\n",
      " [ 0.  0.  0.  0. -1.]\n",
      " [ 0.  2.  0.  0. -1.]\n",
      " [ 0.  0.  0.  0. -1.]\n",
      " [ 0.  0.  0.  0. -1.]]\n",
      "Action 2\n",
      "[[ 0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  2.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.]\n",
      " [ 0.  0.  0.  0.  0.]\n",
      " [-1. -1. -1. -1. -1.]]\n",
      "Action 3\n",
      "[[-1.  0. 10.  0.  5.]\n",
      " [-1.  0.  0.  0.  0.]\n",
      " [-1.  0.  0.  2.  0.]\n",
      " [-1.  0.  0.  0.  0.]\n",
      " [-1.  0.  0.  0.  0.]]\n"
     ]
    }
   ],
   "source": [
    "grid2d.ShowR()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "true"
   },
   "source": [
    "### Find optimal policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "grid2d.ShowValue()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 3.80734393, -1.40504816,  4.89235522,  0.18039256,  1.72683283,\n",
       "        1.82299998,  3.36666558,  2.52054581,  2.124479  ,  0.70742102,\n",
       "        0.21632375,  0.9133494 ,  0.818926  ,  0.47821396, -0.30352832,\n",
       "       -0.88012313, -0.34258469, -0.27243803, -0.51448131, -1.12001031,\n",
       "       -1.79416317, -1.28338684, -1.17269571, -1.37234922, -1.92869254])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid2d.EvaluatePolicy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 3.8 -1.4  4.9  0.2  1.7]\n",
      " [ 1.8  3.4  2.5  2.1  0.7]\n",
      " [ 0.2  0.9  0.8  0.5 -0.3]\n",
      " [-0.9 -0.3 -0.3 -0.5 -1.1]\n",
      " [-1.8 -1.3 -1.2 -1.4 -1.9]]\n"
     ]
    }
   ],
   "source": [
    "grid2d.ShowValue()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Action 0\n",
      "[[0. 0. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 0.]\n",
      " [1. 1. 1. 1. 1.]\n",
      " [1. 1. 1. 1. 1.]\n",
      " [1. 1. 1. 1. 1.]]\n",
      "Action 1\n",
      "[[1.         0.33333333 0.         0.33333333 0.        ]\n",
      " [0.         0.         0.         0.         0.        ]\n",
      " [0.         0.         0.         0.         0.        ]\n",
      " [0.         0.         0.         0.         0.        ]\n",
      " [0.         0.         0.         0.         0.        ]]\n",
      "Action 2\n",
      "[[0.         0.33333333 0.         0.33333333 0.        ]\n",
      " [0.         0.         0.         0.         0.        ]\n",
      " [0.         0.         0.         0.         0.        ]\n",
      " [0.         0.         0.         0.         0.        ]\n",
      " [0.         0.         0.         0.         0.        ]]\n",
      "Action 3\n",
      "[[0.         0.33333333 1.         0.33333333 1.        ]\n",
      " [0.         0.         0.         0.         1.        ]\n",
      " [0.         0.         0.         0.         0.        ]\n",
      " [0.         0.         0.         0.         0.        ]\n",
      " [0.         0.         0.         0.         0.        ]]\n"
     ]
    }
   ],
   "source": [
    "Pi_opt = grid2d.OptimalPolicy()\n",
    "grid2d.Pi = Pi_opt\n",
    "grid2d.ShowPolicy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Action 0\n",
      "[[0.  0.  0.  0.  0. ]\n",
      " [0.5 1.  0.5 0.  0.5]\n",
      " [0.5 1.  0.5 0.  0.5]\n",
      " [0.5 1.  0.5 0.  0.5]\n",
      " [0.5 1.  0.5 0.  0.5]]\n",
      "Action 1\n",
      "[[1.         0.33333333 0.         0.33333333 0.        ]\n",
      " [0.5        0.         0.         0.         0.        ]\n",
      " [0.5        0.         0.         0.         0.        ]\n",
      " [0.5        0.         0.         0.         0.        ]\n",
      " [0.5        0.         0.         0.         0.        ]]\n",
      "Action 2\n",
      "[[0.         0.33333333 0.         0.33333333 0.        ]\n",
      " [0.         0.         0.         0.         0.        ]\n",
      " [0.         0.         0.         0.         0.        ]\n",
      " [0.         0.         0.         0.         0.        ]\n",
      " [0.         0.         0.         0.         0.        ]]\n",
      "Action 3\n",
      "[[0.         0.33333333 1.         0.33333333 1.        ]\n",
      " [0.         0.         0.5        1.         0.5       ]\n",
      " [0.         0.         0.5        1.         0.5       ]\n",
      " [0.         0.         0.5        1.         0.5       ]\n",
      " [0.         0.         0.5        1.         0.5       ]]\n"
     ]
    }
   ],
   "source": [
    "Pi_opt = grid2d.PolicyIteration()\n",
    "grid2d.Pi = Pi_opt\n",
    "grid2d.ShowPolicy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### SARSA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "for n_episodes in range(500):\n",
    "    grid2d.state = np.random.randint(0,grid2d.n_states)\n",
    "    grid2d.SARSA(alpha=0.1, eps=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "for n_episodes in range(500):\n",
    "    grid2d.state = np.random.randint(0,grid2d.n_states)\n",
    "    grid2d.QLearning(alpha=0.1, eps=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Action 0\n",
      "[[ 9.795  8.923 11.019  4.535  5.327]\n",
      " [15.634 17.99  14.116 16.118  4.552]\n",
      " [ 5.831 11.303 14.497 12.951  4.44 ]\n",
      " [ 7.223 10.887 15.009 11.889  6.125]\n",
      " [10.582  9.017 13.44  11.8    6.132]]\n",
      "Action 1\n",
      "[[17.796  4.269 16.023  4.506  4.294]\n",
      " [ 7.675 14.074 14.005  9.946  5.052]\n",
      " [ 6.544 14.924 13.116  7.179  2.918]\n",
      " [12.042 13.457 11.667  9.602  6.502]\n",
      " [ 5.98  11.941  8.227  5.286  4.557]]\n",
      "Action 2\n",
      "[[ 8.155  3.371  9.988  3.67   1.577]\n",
      " [ 8.001 13.163 14.788  8.955  3.894]\n",
      " [10.677 10.29  13.27  10.474 10.438]\n",
      " [ 6.961  9.309 11.77   9.275  6.472]\n",
      " [ 4.152  5.18   9.314  4.328  2.487]]\n",
      "Action 3\n",
      "[[ 9.133  4.894 13.873 12.448 15.373]\n",
      " [ 9.555 13.635 16.145 11.835 14.1  ]\n",
      " [ 4.068  8.024 13.162 14.863  3.984]\n",
      " [ 7.768  9.699 11.665 13.431 11.927]\n",
      " [ 3.464  5.494  8.468  9.758 10.106]]\n"
     ]
    }
   ],
   "source": [
    "grid2d.ShowQ()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Action 0\n",
      "[[0. 1. 0. 0. 0.]\n",
      " [1. 1. 0. 1. 0.]\n",
      " [0. 0. 1. 0. 0.]\n",
      " [0. 0. 1. 0. 0.]\n",
      " [1. 0. 1. 1. 0.]]\n",
      "Action 1\n",
      "[[1. 0. 1. 0. 0.]\n",
      " [0. 0. 0. 0. 0.]\n",
      " [0. 1. 0. 0. 0.]\n",
      " [1. 1. 0. 0. 0.]\n",
      " [0. 1. 0. 0. 0.]]\n",
      "Action 2\n",
      "[[0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0.]\n",
      " [1. 0. 0. 0. 1.]\n",
      " [0. 0. 0. 0. 0.]\n",
      " [0. 0. 0. 0. 0.]]\n",
      "Action 3\n",
      "[[0. 0. 0. 1. 1.]\n",
      " [0. 0. 1. 0. 1.]\n",
      " [0. 0. 0. 1. 0.]\n",
      " [0. 0. 0. 1. 1.]\n",
      " [0. 0. 0. 0. 1.]]\n"
     ]
    }
   ],
   "source": [
    "grid2d.Pi = grid2d.OptimalPolicyFromQ()\n",
    "grid2d.ShowPolicy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[10, 15, 16, 17, 12, 7, 6, 1]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeUAAAHWCAYAAABJ3pFhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAcWklEQVR4nO3dfbBdZX0v8O8jUCAXig4EqRITey1nKK1Nb5lWZFoi0ttI8aXVop1AhxmdOHA7IzMwd4ro9N5R5B/s8E91DLbDiLktGakWKeoVdctUIhVoEFJIJgh5ETCA7mBuIL7kuX/sMHA4ycmWvZv17OzPZ+YMe6+1zrN/82PnfM/zrLX2KbXWAADde1nXBQAAA0IZABohlAGgEUIZABohlAGgEUIZABoxUiiXUv6slLK+lLKnlHL6uIoCgGk06kz5/iR/muT2MdQCAFPt8FG+udb6QJKUUsZTDQBMMeeUAaARB5wpl1JuS3LSPnZdWWv952FfqJSyMsnKJDnqqKN+5zWvec3QRU6rPXv25GUv83vTMPRqOPo0PL0ajj4NZ+PGjU/WWhce6LgDhnKt9ZxxFFRrXZVkVZLMzMzUDRs2jGPYQ1qv18uyZcu6LmMi6NVw9Gl4ejUcfRpOKWXzMMf59QYAGjHqLVF/UkrZluSMJP9SSvnKeMoCgOkz6tXXn0/y+THVAgBTzfI1ADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI4QyADRCKANAI8YSyqWU5aWUDaWUTaWUvxrHmDCM1auTJUuSs88+K0uWDJ4DTKrDRx2glHJYkr9N8odJtiX5Tinl5lrrf4w6Nsxn9epk5cpk164kKdm8efA8SVas6LIygJdmHDPl302yqdb6vVrrT5L8Y5K3j2FcmNeVVz4XyM/btWuwHWASjTxTTvLqJFtf8Hxbkt978UGllJVJVibJwoUL0+v1xvDSh7adO3fq0zy2bDkrSdnH9ppe75sHv6AJ4D01PL0ajj6N1zhCee5PxaTO2VDrqiSrkmRmZqYuW7ZsDC99aOv1etGn/XvNa5LNm/e1vejbfnhPDU+vhqNP4zWO5ettSRa94PnJSR4dw7gwr6uuShYsmL1twYLBdoBJNI5Q/k6SXyulvLaU8ktJ3pPk5jGMC/NasSJZtSo58sgkqVm8ePDcRV7ApBp5+brW+rNSyl8m+UqSw5L8fa11/ciVwRBWrEiuuy7p93dk3bqXd10OwEjGcU45tdZbk9w6jrHgF/WpTyV33rkh+7i+EGCijCWUoUszM8ljjz3TdRkAI/Mxm0y8L34xueOO47suA2BkQpmJ9/GPJ2vWLDrwgQCNE8oA0AihDACNEMoA0AihDACNcEsUE++GG5K1ax9IckbXpQCMxEyZibdoUXLiibu7LgNgZEKZiXfjjcnXv76w6zIARiaUmXif/GRy882v7roMgJEJZQBohFAGgEYIZQBohFAGgEa4T5mJ97nPJd/61vokZ3ZdCsBIzJSZeCeckBx33E+7LgNgZEKZiXf99cmXv3xS12UAjEwoM/GEMnCoEMoA0AihDACNEMoA0AihDACNcJ8yE+/WW5Pbb/9ukj/ouhSAkZgpM/EWLEiOOmpP12UAjEwoM/E+8YnkC194VddlAIzM8jUTb82apN8/sesyAEZmpgwAjRDKANAIoQwAjRDKANAIF3ox8Xq9pNdbl2RZx5UAjMZMGQAaIZSZeNdck9x446KuywAYmeVrJt4ttyT9/vFdlwEwMjNlAGiEUAaARghlAGiEUGbiHX10cuSRP++6DICRudCLifelLyW93n1xnzIw6cyUAaARQpmJ95GPJJ/5zOKuywAYmeVrJt7Xvpb0+6/ougyAkZkpA0AjhDIANEIoA0AjnFNm4h1/fLJnz0+7LgNgZEKZiXfTTUmvtz7uUwYmneVrAGiEmTIT74orki1bXptly7quBGA0QpmJt3Zt0u8f13UZACOzfA0AjRDKANAIoQwAjXBOmYl38snJEUfs7roMgJEJZSbeZz+b9HoPJHll16UAjMTyNQA0wkyZiXfppcm2ba9znzIw8YQyE2/duqTfP6brMgBGZvkaABohlAGgEUIZABohlJl4p5ySnHzyrq7LABjZWC70KqX8fZLzkmyvtf7GOMaEYa1alfR6G5O8qutSAEYyrpny9UmWj2ksGNrq1cmSJcnZZ5+VJUsGz5lLn2AyjGWmXGu9vZSyZBxjwbBWr05Wrkx27UqSks2bB8+TZMWKLitriz7B5HCfMhPryiufC5rn7dqVXHzx82FzwQXJtm2zjznjjOTqqweP3/nO5KmnZu9/85uTD3948Pgtb0meeWb2/vPOSy6/fPB4Xx9Ycv75ySWXDGo599y5+y+6aPD15JPJu941d//FFyfvfneydWty4YVz9192WfLWtyYbNiTvf//c/R/6UHLOOYP7ty+9NPn2t5PdL/po8F27Bv0TytCWgxbKpZSVSVYmycKFC9Pr9Q7WS0+snTt36tM8tmw5K0mZs/3HP67p9b6ZJPnBD05Nv3/ki75vR3q9h5MkTzxxWp5++ohZ+x9++Efp9TYnSX74w9/M7t2Hzdr/0ENPpdfbmiTp95fOef2NG7en13s0zz77svT7r5+z/8EHH0+v93h27Dgi/f5pc/avX//99HpPZPv2I9Pvnzpn/333bc2xxz6VLVuOTr8/M2f/vfduzuGH/yibNh2Tfv912b37uOyrT1u2PN8n5vLvbzj6NF6l1jqegQbL17cMc6HXzMxM3bBhw1he91DW6/WyzGdH7teSJcnmzXO3L16cPPLIwa6mXfr00vj3Nxx9Gk4p5e5a6+kHOs4tUUysq65KFiyYvW3BgsF2nqdPMDnGEsqllH9IsjbJTCllWynlveMYF+azYsXgdqjFi5NSahYvHjx3nnS25/r0ylcmiT5By8Z19fWfj2Mc+EWtWDH46vW+aQltHitWJNddl5x00o6sW/fyrssB9sPyNQA0QigDQCOEMgA0QigDQCN8ohdMiWuvTe66a1OSA94qCXREKMOUWLo06fd3dl0GMA/L1zAlbrstufvuV3RdBjAPoQxT4qMfTW64YXHXZQDzEMoA0AihDACNEMoA0AihDACNcEsUTIlPfSq5884NSX6v61KA/RDKMCVmZpLHHnum6zKAeVi+hinxxS8md9xxfNdlAPMQyjAlPv7xZM2aRV2XAcxDKANAI4QyADRCKANAI4QyADTCLVEwJW64IVm79oEkZ3RdCrAfZsowJRYtSk48cXfXZQDzEMowJW68Mfn61xd2XQYwD6EMU+KTn0xuvvnVXZcBzEMoA0AjhDIANEIoA0AjhDIANMJ9yjAlPve55FvfWp/kzK5LAfbDTBmmxAknJMcd99OuywDmIZRhSlx/ffLlL5/UdRnAPIQyTAmhDO0TygDQCKEMAI0QygDQCKEMAI1wnzJMiVtvTW6//btJ/qDrUoD9MFOGKbFgQXLUUXu6LgOYh1CGKfGJTyRf+MKrui4DmIfla5gSa9Yk/f6JXZcBzMNMGQAaIZQBoBFCGQAaIZQBoBEu9IIp0eslvd66JMs6rgTYHzNlAGiEUIYpcc01yY03Luq6DGAelq9hStxyS9LvH991GcA8zJQBoBFCGQAaIZQBoBFCGabE0UcnRx75867LAObhQi+YEl/6UtLr3Rf3KUO7zJQBoBFCGabERz6SfOYzi7suA5iH5WuYEl/7WtLvv6LrMoB5mCkDQCOEMgA0QigDQCOcU4YpcfzxyZ49P+26DGAeQhmmxE03Jb3e+rhPGdpl+RoAGmGmDFPiiiuSLVtem2XLuq4E2B+hDFNi7dqk3z+u6zKAeYy8fF1KWVRK+UYp5YFSyvpSygfGUdg0W706WbIkOfvss7JkyeA5jGL16uTb307uvfc47ylo2Dhmyj9Lclmt9Z5SyrFJ7i6lfLXW+h9jGHvqrF6drFyZ7NqVJCWbNw+eJ8mKFV1WxqR67j21e3fiPQVtG3mmXGt9rNZ6z97HP07yQJJXjzrutLryyucC+Xm7dg22w0vhPQWTY6znlEspS5L8dpI797FvZZKVSbJw4cL0er1xvvQhY8uWs5KUfWyv6fW+efALmhA7d+70ntoP76mXxntqOPo0XqXWOp6BSjkmyTeTXFVr/af5jp2ZmakbNmwYy+seapYsSTZvnrt98eLkkUcOdjWTo9frZZnLivfJe+ql8Z4ajj4Np5Ryd6319AMdN5b7lEspRyS5KcnqAwUy87vqqmTBgtnbFiwYbIeXwnsKJsc4rr4uSf4uyQO11r8ZvaTptmJFsmpVcuSRSVKzePHguQtyeKmee08de2ziPQVtG8c55TOTXJjkvlLKur3bPlhrvXUMY0+lFSuS665L+v0dWbfu5V2XwyHAewomw8ihXGv91+zrKhJGcv75ycaN25P4AQowLXz2daMuuSR5xzse7boMAA4iodyoXbuSZ5/1vwdgmvjs60ade27S778+y5d3XQmHilNOSR59dFecEoF2CWWYEqtWJb3exiSv6roUYD+sjwJAI8yUYUqsXJk8+ugp/p4yNEwow5TYuDHp9xcc+ECgM0K5URddlDz44ONxUQ7A9HBOuVEXXZQsX/5412UAcBAJ5UY9+WSyY8cRXZcBwEFk+bpR73pX0u+flre/vetKOFQsXZps27YzTolAu4QyTIlrr016vU1JTu66FGA/LF8DQCPMlGFKXHBB8oMfnOo+ZWiYUIYpsW1b0u8f2XUZwDyEcqMuvjhZv/77cVEOwPQQyo1697uTXu+JrssA4CByoVejtm5Ntm+31AgwTcyUG3XhhUm/f2rOP7/rSjhUnHFGsmXLjjglAu0SyjAlrr466fUeTrK461KA/bB8DQCNMFOGKfHOdyZPPHFabr+960qA/TFThinx1FPJ00/7IyfQMjPlRl12WXLffVvjohyA6SGUG/XWtybHHvtU12UAcBBZvm7Uhg3Jli1Hd10GAAeRmXKj3v/+pN+fyV/8RdeVcKh485uThx/+UZwSgXYJZZgSH/5w0uttTvLarksB9sPyNQA0wkwZpsRb3pL88Ie/mTvv7LoSYH/MlGFKPPNMsnv3YV2XAczDTLlRH/pQcu+9m+OiHIDpIZQbdc45yeGH/6jrMgA4iCxfN2rdumTTpmO6LgOAg8hMuVGXXpr0+6/L+97XdSUcKs47L3nooafilAi0SyjDlLj88qTX25rkv3ZdCrAflq8BoBFmyjAlli1L+v2lWbeu60qA/TFTBoBGmCk36mMfS+6553tJ/lvXpQBwkAjlRr3xjclPfvJ012UAcBBZvm7UHXck99//y12XAcBBJJQb9cEPJp/+9K92XQaHkPPPT5Yt2951GcA8hDJMiUsuSd7xjke7LgOYh1CGKbFrV/Lss/7JQ8tc6AVT4txzk37/9Vm+vOtKgP3xazMANMJMuVHXXpvcddemJKd3XQoAB4lQbtTSpUm/v7PrMgA4iCxfN+q225K7735F12UAcBAJ5UZ99KPJDTcs7roMDiEXXZQsX/5412UA8xDKMCWEMrRPKMOUePLJZMeOI7ouA5iHC71gSrzrXUm/f1re/vauKwH2x0wZABphptyoT30qufPODUl+r+tSADhIhHKjZmaSxx57pusyADiILF836otfTO644/iuywDgIBLKjfr4x5M1axZ1XQaHkIsvTt72tu93XQYwD8vXMCXe/e6k13ui6zKAeZgpw5TYujXZvv3IrssA5mGmDFPiwguTfv/UnH9+15UA+2OmDACNGHmmXEo5KsntSY7cO97naq1/Peq40+6GG5K1ax9IckbXpQBwkIxjprw7ydm11t9KsjTJ8lLKG8Yw7tRavTr5/d9P3vOeN2TJksFz9m316mTJkuTss8/Sq3msXp18+9vJvfcep0/QsJFnyrXWmmTn3qdH7P2qo447rVavTlauTHbtSpKSzZsHz5NkxYouK2uPXg3nuT7t3p3oE7RtLBd6lVIOS3J3ktcl+dta653jGHcaXXnlcyHzvF27kve+N/mjP0pOOCG5/vrB14vdemuyYEHyiU8ka9bM3d/rDf57zTXJLbfM3nf00cmXvjR4/JGPJF/72uz9xx+f3HTT4PEVVyRr187ef/LJyWc/O3h86aXJunWz959ySrJq1eDxypXJxo2z9y9dmlx77eDxBRck27bN3n/GGcnVVw8ev/OdyVNPDWZ+g6B53q5dyfvel1x33ezt552XXH754PGyZZnj/POTSy4ZfP+5587df9FFg68nnxz8YYcXu/jiwS1HW7cOLqh6scsuS9761mTDhuT975+7/0MfSs45Z9C3Sy+du/9jH0ve+MbkjjuSD35w7v5rrx308LbbBn+L+4X216crrxTK0JqxhHKt9edJlpZSXp7k86WU36i13v/CY0opK5OsTJKFCxem91xCMMuWLWclKXO2795d861v3ZHjjvtpHnzwpPT7J8055vbbv5ujjtqTjRtflX7/xDn7e71BUj700KL0+7M/LeyZZ36eXu++JMnDDy9Ov/+KWfv37Plper31e2t8bfr942btP+KI3en1HkiSbNv2uvT7x8za/+iju9Lrbdz7+JT0+wtm7d+2bWd6vU1Jkh/84NT0+7Nv3dmyZUd6vYeTJE88cVqefvqI7N59XPbVq2efren3d8za9tBDT6XX25ok6feXzvmejRu3p9d7NM8++7L0+6+fs//BBx9Pr/d4duw4Iv3+aXP2r1///fR6T2T79iPT7586Z/99923Nscc+lS1bjk6/PzNn/733bs7hh/8omzYdk37/dXP233PP9/KTnzyd++//5fT7vzpn/113bUq/vzP33vuK9PuLZ+3bX5+2bKnp9b45ZzsDO3fu9HNqCPo0XmWw+jzGAUv56yT/r9Z6zf6OmZmZqRs2bBjr6x4qlixJNm+eu33x4uSRRw52NW3Tq+Ho00vT6/WybF/LKsyiT8Mppdxdaz39QMeNfKFXKWXh3hlySilHJzknyYOjjjutrrpqsAT9QgsWDLYzm14NR59gcozj6utfSfKNUsp3k3wnyVdrrbcc4HvYjxUrBudeFy9OSqlZvHjw3Lm/ufRqOPoEk2Psy9fDsHw9HMtCw9Or4ejT8PRqOPo0nIO2fA0AjIdQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGCGUAaIRQBoBGjC2USymHlVL+vZRyy7jGBIBpMs6Z8geSPDDG8QBgqowllEspJyf54ySfHsd4ADCNSq119EFK+VySq5Mcm+TyWut5+zhmZZKVSbJw4cLfWbNmzcive6jbuXNnjjnmmK7LmAh6NRx9Gp5eDUefhvOmN73p7lrr6Qc67vBRX6iUcl6S7bXWu0spy/Z3XK11VZJVSTIzM1OXLdvvoezV6/WiT8PRq+Ho0/D0ajj6NF7jWL4+M8nbSimPJPnHJGeXUj47hnEBYKqMHMq11itqrSfXWpckeU+Sr9daLxi5MgCYMu5TBoBGjHxO+YVqrb0kvXGOCQDTwkwZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABohlAGgEUIZABpx+DgGKaU8kuTHSX6e5Ge11tPHMS4ATJOxhPJeb6q1PjnG8QBgqli+BoBGjCuUa5L/W0q5u5SyckxjAsBUGdfy9Zm11kdLKScm+Wop5cFa6+0vPGBvWD8X2LtLKfeP6bUPZSckcUpgOHo1HH0anl4NR5+GMzPMQaXWOtZXLaX8ryQ7a63XzHPMXS4GOzB9Gp5eDUefhqdXw9Gn4Qzbp5GXr0sp/6WUcuxzj5P89yRmwQDwCxrH8vUrk3y+lPLceP+n1vrlMYwLAFNl5FCutX4vyW/9gt+2atTXnRL6NDy9Go4+DU+vhqNPwxmqT2M/pwwAvDTuUwaARnQWyqWUPyulrC+l7CmluHLvRUopy0spG0opm0opf9V1Pa0qpfx9KWW7W+zmV0pZVEr5Rinlgb3/7j7QdU0tKqUcVUr5t1LKvXv79L+7rqllpZTDSin/Xkq5petaWlZKeaSUcl8pZV0p5a75ju1ypnx/kj9NcvuBDpw2pZTDkvxtkrck+fUkf15K+fVuq2rW9UmWd13EBPhZkstqracmeUOS/+E9tU+7k5xda/2tJEuTLC+lvKHjmlr2gSQPdF3EhHhTrXXpgW6L6iyUa60P1Fo3dPX6jfvdJJtqrd+rtf4kyT8meXvHNTVp74fU/LDrOlpXa32s1nrP3sc/zuAH6au7rao9dWDn3qdH7P1y4c0+lFJOTvLHST7ddS2HEueU2/TqJFtf8Hxb/ABlTEopS5L8dpI7u62kTXuXZNcl2Z7kq7VWfdq3a5P8zyR7ui5kAgz9UdTj/CtRc5RSbkty0j52XVlr/ef/zNeecGUf2/y2zshKKcckuSnJpbXWp7uup0W11p8nWVpKeXkGn8HwG7VW1yy8QCnlvCTba613l1KWdV3PBDjgR1E/5z81lGut5/xnjn8I25Zk0Quen5zk0Y5q4RBRSjkig0BeXWv9p67raV2ttV9K6WVwzYJQnu3MJG8rpZyb5Kgkv1xK+Wyt9YKO62pSrfXRvf/dXkr5fAanKPcZypav2/SdJL9WSnltKeWXkrwnyc0d18QEK4OP3Pu7JA/UWv+m63paVUpZuHeGnFLK0UnOSfJgt1W1p9Z6Ra315Frrkgx+Pn1dIO/bL/pR1F3eEvUnpZRtSc5I8i+llK90VUtraq0/S/KXSb6SwQU5a2qt67utqk2llH9IsjbJTCllWynlvV3X1Kgzk1yY5Oy9t2Ws2zvLYbZfSfKNUsp3M/jl+Ku1Vrf7MIpXJvnXUsq9Sf4tyb/M91HUPtELABph+RoAGiGUAaARQhkAGiGUAaARQhkAGiGUAaARQhkAGiGUAaAR/x9ggCRR9G8jJQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "grid2d.state = coord2idx([2,0], 5)\n",
    "traj = grid2d.Episode()\n",
    "print(traj)\n",
    "\n",
    "grid2d.PlotTrajectory(traj)\n",
    "plt.gca().invert_yaxis()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A_coords in [(1,1), (2,3), (0,1)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## 2D GridWorld with Holes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "true"
   },
   "source": [
    "### Set up the matrices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "gl = 5  # grid length\n",
    "gs = gl*gl\n",
    "n_actions = 4  # number of actions\n",
    "A_coords = (0,1)\n",
    "A_reward = 10.\n",
    "Ap_coords = (4,1)\n",
    "B_coords = (0,3)\n",
    "B_reward = 5\n",
    "Bp_coords = (3,3)\n",
    "#C_coords = (2,2)\n",
    "#C_reward = 2\n",
    "holes = [(2,2),(2,3),(1,2)]\n",
    "hole_reward = -7.\n",
    "terminal = copy.deepcopy(holes)\n",
    "terminal.append(A_coords)\n",
    "terminal.append(B_coords)\n",
    "movement_cost = 0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "def coord2idx(coords, row_length):\n",
    "    return coords[0]*row_length + coords[1]\n",
    "def idx2coord(idx, row_length):\n",
    "    row = idx//row_length\n",
    "    col = idx%row_length\n",
    "    return (row,col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# State Transition Probabilities: 2D Grid\n",
    "# P[s,a,t] is the probability of action a taking you from state s to state t\n",
    "# P[s,a,t] = p(t|a,s)\n",
    "# 0==up, 1==right, 2==down, 3==left\n",
    "P = np.zeros((gs,n_actions,gs))\n",
    "for sr in range(gl):\n",
    "    sr_offset = sr*gl\n",
    "    for sc in range(gl):\n",
    "        #idx = sr_offset + sc\n",
    "        idx = coord2idx([sr,sc], gl)\n",
    "        for a in range(4):\n",
    "            img = np.zeros((gl,gl))\n",
    "            if (sr,sc)==A_coords:   # A, any action\n",
    "                img[Ap_coords[0],Ap_coords[1]] = 1.\n",
    "            elif (sr,sc)==B_coords: # B, any action\n",
    "                img[Bp_coords[0],Bp_coords[1]] = 1.\n",
    "            elif (sr,sc) in holes:\n",
    "                img[sr,sc] = 1.\n",
    "            elif sr==0 and a==0:    # top row, up\n",
    "                img[sr,sc] = 1.\n",
    "            elif sr==gl-1 and a==2: # bottom row, down\n",
    "                img[sr,sc] = 1.\n",
    "            elif sc==0 and a==3:    # left col, left\n",
    "                img[sr,sc] = 1.\n",
    "            elif sc==gl-1 and a==1: # right col, right\n",
    "                img[sr,sc] = 1.\n",
    "            elif a==0:              # up\n",
    "                img[sr-1,sc] = 1.\n",
    "            elif a==1:              # right\n",
    "                img[sr,sc+1] = 1.\n",
    "            elif a==2:              # down\n",
    "                img[sr+1,sc] = 1.\n",
    "            elif a==3:              # left\n",
    "                img[sr,sc-1] = 1.\n",
    "            P[idx,a,:] = img.flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Reward Function: 2D Grid\n",
    "# R[s,a] is the reward of taking action a from state s to state t\n",
    "# R[s,a] = r(s,a)\n",
    "R = np.zeros((gs,n_actions))\n",
    "for sr in range(gl):\n",
    "    sr_offset = sr*gl\n",
    "    for sc in range(gl):\n",
    "        #idx = sr_offset + sc\n",
    "        idx = coord2idx([sr,sc], gl)\n",
    "        #svec = np.zeros(gs)\n",
    "        #svec[idx] = 1.\n",
    "        for a in range(4):\n",
    "            reward = 0.\n",
    "            tvec = P[idx,a,:]\n",
    "            tidx = np.argmax(tvec)\n",
    "            tcoords = idx2coord(tidx, gl)\n",
    "            if tcoords==A_coords:\n",
    "                reward += A_reward\n",
    "            elif tcoords==B_coords:\n",
    "                reward += B_reward\n",
    "            #elif tcoords==C_coords:\n",
    "            #    reward += C_reward\n",
    "            elif tcoords in holes:\n",
    "                reward += hole_reward\n",
    "            elif sr==0 and a==0:    # top row, up\n",
    "                reward += -1.\n",
    "            elif sr==gl-1 and a==2: # bottom row, down\n",
    "                reward += -1.\n",
    "            elif sc==0 and a==3:    # left col, left\n",
    "                reward += -1.\n",
    "            elif sc==gl-1 and a==1: # right col, right\n",
    "                reward += -1.\n",
    "            R[idx,a] = reward"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "### Create the MDP object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 527,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "grid2d = MDP_2d(P, R, terminal=terminal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# SARSA\n",
    "for n_episodes in range(50):\n",
    "    grid2d.state = np.random.randint(0,grid2d.n_states)\n",
    "    grid2d.SARSA(alpha=0.1, eps=0.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 457,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# SARSA(lambda)\n",
    "for n_episodes in range(50):\n",
    "    grid2d.state = np.random.randint(0,grid2d.n_states)\n",
    "    grid2d.SARSA_lambda(alpha=0.1, eps=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 540,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Dyna-Q\n",
    "for n_episodes in range(50):\n",
    "    grid2d.state = np.random.randint(0,grid2d.n_states)\n",
    "    grid2d.DynaQ(alpha=0.1, eps=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Q-Learning\n",
    "for n_episodes in range(500):\n",
    "    grid2d.state = np.random.randint(0,grid2d.n_states)\n",
    "    grid2d.QLearning(alpha=0.1, eps=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 541,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Action 0\n",
      "[[-1.000e-01  0.000e+00 -1.000e-01 -9.000e-02  4.100e-02]\n",
      " [ 1.400e-02  9.657e+00 -7.000e-01  0.000e+00  7.680e-01]\n",
      " [ 0.000e+00  7.682e+00 -1.330e+00  0.000e+00  8.800e-02]\n",
      " [ 0.000e+00  4.558e+00 -3.652e+00 -2.407e+00  3.000e-03]\n",
      " [ 3.000e-03  2.092e+00  1.470e-01  3.000e-02  0.000e+00]]\n",
      "Action 1\n",
      "[[ 0.     0.     0.     0.     0.   ]\n",
      " [ 2.946 -1.897 -1.33   0.034 -0.26 ]\n",
      " [ 2.821 -3.987  0.    -0.7   -0.19 ]\n",
      " [ 0.     0.324  0.029  0.    -0.1  ]\n",
      " [ 0.879  0.021  0.     0.    -0.344]]\n",
      "Action 2\n",
      "[[ 0.481  0.     0.     0.019  0.   ]\n",
      " [ 0.418  2.233  0.    -0.7    0.   ]\n",
      " [ 0.     0.388 -1.33   0.     0.   ]\n",
      " [ 0.169  0.294  0.008  0.     0.   ]\n",
      " [-0.163  0.064 -0.1   -0.19  -0.1  ]]\n",
      "Action 3\n",
      "[[-0.158  0.     1.9    0.     2.855]\n",
      " [ 0.266  0.908 -0.7   -1.897  0.   ]\n",
      " [ 0.     0.078  0.    -0.7   -1.33 ]\n",
      " [-0.271  0.01   1.734  0.304  0.014]\n",
      " [-0.308  0.141  0.324  0.003  0.   ]]\n"
     ]
    }
   ],
   "source": [
    "grid2d.ShowQ()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 542,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Action 0\n",
      "[[0.   0.25 0.   0.   0.  ]\n",
      " [0.   1.   0.   0.   1.  ]\n",
      " [0.   1.   0.   0.5  1.  ]\n",
      " [0.   1.   0.   0.   0.  ]\n",
      " [0.   1.   0.   1.   0.5 ]]\n",
      "Action 1\n",
      "[[0.   0.25 0.   0.   0.  ]\n",
      " [1.   0.   0.   1.   0.  ]\n",
      " [1.   0.   0.5  0.   0.  ]\n",
      " [0.   0.   0.   0.   0.  ]\n",
      " [1.   0.   0.   0.   0.  ]]\n",
      "Action 2\n",
      "[[1.   0.25 0.   1.   0.  ]\n",
      " [0.   0.   1.   0.   0.  ]\n",
      " [0.   0.   0.   0.5  0.  ]\n",
      " [1.   0.   0.   0.   0.  ]\n",
      " [0.   0.   0.   0.   0.  ]]\n",
      "Action 3\n",
      "[[0.   0.25 1.   0.   1.  ]\n",
      " [0.   0.   0.   0.   0.  ]\n",
      " [0.   0.   0.5  0.   0.  ]\n",
      " [0.   0.   1.   1.   1.  ]\n",
      " [0.   0.   1.   0.   0.5 ]]\n"
     ]
    }
   ],
   "source": [
    "grid2d.Pi = grid2d.OptimalPolicyFromQ()\n",
    "grid2d.ShowPolicy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 543,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[18, 17, 16, 11, 6, 1]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeUAAAHWCAYAAABJ3pFhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAZ0klEQVR4nO3df7DddX3n8deHSyK3BMUfsYWLP6qyt4IoKQzqxHYCugu71ZYydbRdneJ2Nu3OarV1UdPO6No/Ci2dtlO30zZtlY60KlJMxbZk1XgHddBIuEAIISC/Qm5YgcIRLt6Q5OazfyRYQn4dOUfP55s8HjMMyTkn3/PmM4fzzPec8zm31FoDAIzeUaMeAADYTZQBoBGiDACNEGUAaIQoA0AjRBkAGjFQlEspby2lbCil7CqlnDmsoQDgSDTomfItSS5Icu0QZgGAI9rRg/zhWuvGJCmlDGcaADiCeU8ZABpxyDPlUsqXkvzEfq763VrrP/V7R6WU5UmWJ8kxxxxzxotf/OK+hzxS7dq1K0cd5e9N/bBW/bFO/bNW/bFO/bn99tsfqrUuPtTtDhnlWuubhjFQrXVlkpVJMjk5WTdt2jSMwx7WpqamsmzZslGP0QnWqj/WqX/Wqj/WqT+llHv7uZ2/3gBAIwbdEvWLpZQtSV6f5J9LKauHMxYAHHkG/fT155J8bkizAMARzcvXANAIUQaARogyADRClAGgEaIMAI0QZQBohCgDQCNEGQAaIcoA0AhRBoBGiDIANEKUAaARogwAjRBlAGiEKANAI0QZABohygDQCFEGgEaIMgA0QpQBoBGiDACNEGUAaIQoA0AjRBkAGiHKANAIUQaARogyADRClAGgEaIMAI0QZQBohCgDQCNEGQAaIcoA0AhRBoBGiDIANEKUAaARogwAjRBlAGiEKANAI0QZABohygDQCFEGgEaIMgA0QpQBoBGiDACNEGUAaIQoA0AjRBkAGiHKANAIUQaARogyADRClAGgEaIMAI0QZQBohCgDQCNEGQAaIcoA0AhRBoBGiDIANEKUAaARogwAjRBlAGiEKANAI0QZABoxlCiXUs4rpWwqpXy7lPKhYRwT+rFqeiZLL1mTC695PEsvWZNV0zOjHgngGTt60AOUUsaS/HmS/5hkS5JvlVI+X2u9ddBjw8Gsmp7JiqvWZ27HfJJkpjeXFVetT5Kcv2RilKMBPCPDOFM+K8m3a6131Vq3J/l0kl8YwnHhoC5dven7QX7S3I75XLp604gmAhjMwGfKSSaS3PeU329J8tqn36iUsjzJ8iRZvHhxpqamhnDXh7fZ2VnrdBAzvbkDXm7d9s9jqn/Wqj/WabiGEeWyn8vqPhfUujLJyiSZnJysy5YtG8JdH96mpqZinQ5s4htr9hvmiePHrdsBeEz1z1r1xzoN1zBevt6S5EVP+f1JSbYO4bhwUBedO5nxBWN7XTa+YCwXnTs5ookABjOMKH8rycmllJ8spSxM8vYknx/CceGgzl8ykYsvOC0vWLQwye4z5IsvOM2HvIDOGvjl61rrzlLKu5OsTjKW5OO11g0DTwZ9OH/JRD61dnOev6CX1R88Z9TjAAxkGO8pp9b6L0n+ZRjHAoAjlW/0AoBGiDIANEKUAaARokznffgtp+RXXrlw1GMADGwoH/SCUTr1xOfkwWePHfqGAI1zpkznfe2Oh7LhoflD3xCgcaJM531szR35/J3bRz0GwMBEGQAaIcoA0AhRBoBGiDIANEKU6bzfv+C0XHjqs0Y9BsDARJnOe/niRTlhkYcy0H2eyei8L936nUw/sHPUYwAMTJTpvL/+6l255u4dox4DYGCiDACNEGUAaIQoA0AjRBkAGiHKdN6fvO30LH+1fcpA94kynXfi8eN5/riHMtB9nsnovKtv2ppv3m+fMtB9okznXf6Ne7Nms33KQPeJMgA0QpQBoBGiDACNEGUAaIQo03l/8Y4z8u4lx4x6DICBiTKd97xjF+a4hWXUYwAMTJTpvM9ef1++usWWKKD7jh71ADCoK9dtSa/ny0OA7nOmDACNEGUAaIQoA0AjRBkAGiHKdN5l7zorv32mfcpA94kynTe+cCzPGrNPGeg+UabzPnndPfmyH90IHAbsU6bzvnDz/fYpA4cFZ8oA0AhRBoBGiDIANEKUAaARokznfebXX58Vrx0f9RgAAxNlAGiEKNN5K6+9M/96t33KQPfZp0znfXnjA/YpA4cFZ8oA0AhRBoBGiDIANEKU6bxjFoxloZ8SBRwGRJnO+7v/dlbe7+cpA4cBUQaARogynfdnX74j//Tt7aMeA2Bg9inTeV//9kPp9eZHPQbAwJwpA0AjRBkAGiHKANAI7ynTec/9sYXZ+T37lIHuc6ZM5/3lO8/Ie5bYpwx0nygDQCNEmc77g2tuy2c32acMdJ/3lOm8G+59xD5l4LAwlDPlUsrHSykPlFJuGcbxAOBINKyXry9Lct6QjgV9WzU9k+nNvWx6ZFeWXrImq6ZnRj0SwDM2lCjXWq9N8vAwjgX9WjU9kxVXrc/2+V1JkpneXFZctV6Ygc7yQS8669LVmzK3Y+/3kud2zOfS1ZtGNBHAYH5kH/QqpSxPsjxJFi9enKmpqR/VXXfW7OysdTqImd7cAS+3bvvnMdU/a9Uf6zRcP7Io11pXJlmZJJOTk3XZsmU/qrvurKmpqVinA5v4xpr9hnni+HHrdgAeU/2zVv2xTsPl5Ws666JzJzO+YGyvy8YXjOWicydHNBHAYIa1JepTSa5LMllK2VJK+bVhHBcO5vwlE7n4gtOycGz3w3ji+PFcfMFpOX/JxIgnA3hmhvLyda31l4dxHPhBnb9kIp9auzm9Xi+rP3jOqMcBGIiXrwGgEaIMAI0QZTrvZYuPzU8c66EMdJ8fSEHnXXzBqzM15QvlgO5zegEAjRBlOm/FVTfnE7c8MeoxAAbm5Ws6764HH0/v8V2jHgNgYM6UAaARogwAjRBlAGiE95TpvFNOfHa27Hp01GMADMyZMp33kbecmv/6ymeNegyAgYkyADRClOm89316On9107ZRjwEwMO8p03n3f3dbetvqqMcAGJgzZQBohCgDQCNEGQAa4T1lOu+nX/LcbM5jox4DYGDOlOm8D573U3nr5MJRjwEwMFEGgEaIMp33G59cl49N26cMdJ8o03mPfG97Zrfbpwx0nygDQCNEGQAaIcoA0AhRpvOWvuIFOeX5Y6MeA2Bgokzn/eYbT84vvMI+ZaD7RBkAGuFrNum8X/342jz88LYsWzbqSQAG40yZztu2Yz7b5+1TBrpPlAGgEaIMAI0QZQBohCjTeW985Qtz+gt9ZhHoPs9kdN7yn315pnbdN+oxAAbmTBkAGuFMmc57219dl15vzj5loPOcKQNAI0QZABohygDQCFEGgEaIMp335lefkLNO8JlFoPtEmc575+tfmje+eMGoxwAYmCjTeXPb5/OEnxIFHAZEmc678BNr88fXbxv1GAADE2UAaIQoA0AjRBkAGiHKANAIUabzfumMk/KGCfuUge4TZTrvrWe+KD9zkn3KQPeJMp338OPb89h2+5SB7hNlOu9/XL4u/2faPmWg+0QZABohygDQCFEGgEaIMgA0wuZOOu8dr3tJbr318VGPATAwUabz3vKaE3PcI7ePegyAgXn5ms7b2pvLv83tGvUYAAMTZTrvtz5zY1be/MSoxwAYmCgDQCNEGQAaMXCUSykvKqV8pZSysZSyoZTy3mEMBv1YNT2T6c29bHpkV5ZesiarpmdGPRLAMzaMT1/vTPL+WusNpZTjkqwrpXyx1nrrEI4NB7RqeiYrrlqf7fO7P+Q105vLiqvWJ0nOXzIxytEAnpGBz5RrrffXWm/Y8+vHkmxM4hmRH7pLV2/K3I75vS6b2zGfS1dvGtFEAIMptQ7vR96VUl6a5Nokr6q1Pvq065YnWZ4kixcvPuOKK64Y2v0ermZnZ7No0aJRj9GsC6858BeGXHbesT/CSbrDY6p/1qo/1qk/Z5999rpa65mHut3QvjyklLIoyT8med/Tg5wktdaVSVYmyeTkZF22bNmw7vqwNTU1Fet0YBPfWJOZ3ty+lx8/bt0OwGOqf9aqP9ZpuIby6etSyoLsDvLf11qvGsYx4VAuOncy4wvG9rpsfMFYLjp3ckQTAQxm4DPlUkpJ8rdJNtZa/3jwkaA/T36Y6wNX3pzt87sycfx4Ljp30oe8gM4axsvXS5O8M8n6UsqNey77nVrrvwzh2HBQ5y+ZyKfWbk6v18vqD54z6nEABjJwlGutX0tShjALABzRfKMXADRClOm895xzcn7+5QtHPQbAwPw8ZTrvDSe/IDtnxg59Q4DGOVOm8zZs/W7ufXT+0DcEaJwo03m/d/Wt+YeN20c9BsDARBkAGiHKANAIUQaARogyADRClOm8D5w3mV/6D/YpA91nnzKdd8ZLnpfH7rZPGeg+Z8p03rp7H84dj9inDHSfKNN5f3jNplx5u33KQPeJMgA0QpQBoBGiDACNEGUAaIQo03kffssp+ZVX2qcMdJ99ynTeqSc+Jw8+2z5loPucKdN5X7vjoWx4yD5loPtEmc772Jo78vk77VMGuk+UAaARogwAjRBlAGiEKANAI0SZzvv9C07Lhac+a9RjAAxMlOm8ly9elBMWeSgD3eeZjM770q3fyfQDO0c9BsDARJnO++uv3pVr7t4x6jEABibKANAIUQaARogyADRClAGgEaJM5/3J207P8lfbpwx0nyjTeSceP57nj3soA93nmYzOu/qmrfnm/fYpA90nynTe5d+4N2s226cMdJ8oA0AjRBkAGiHKANAIUQaARogynfcX7zgj715yzKjHABiYKNN5zzt2YY5bWEY9BsDARJnO++z19+WrW2yJArrv6FEPAIO6ct2W9Hq+PAToPmfKANAIUQaARogyADRClAGgEaJM5132rrPy22fapwx0nyjTeeMLx/KsMfuUge4TZTrvk9fdky/70Y3AYcA+ZTrvCzffb58ycFhwpgwAjRBlAGiEKANAI0QZABohynTeZ3799Vnx2vFRjwEwMFEGgEaIMp238to7869326cMdJ99ynTelzc+YJ8ycFhwpgwAjRBlAGjEwFEupRxTSllbSrmplLKhlPLRYQwGAEeaYZwpP5HknFrra5KcnuS8UsrrhnBcOKRV0zO58b5eNj2yK0svWZNV0zOjHqlJq6ZnsvSSNbnwmset0yFYK0Zp4A961Vprktk9v12w55866HHhUFZNz2TFVevzxM5dSZKZ3lxWXLU+SXL+kolRjtaUJ9dpbsd8Eut0MNaKURvKp69LKWNJ1iV5RZI/r7V+cxjHhYO5dPWm7z95Pmlux3w+cOXN+dTazXnujy3MX77zjCTJH1xzW26495G9bnvCc47Jn759SZLko1dvyK1bH93r+pctPjYXX/DqJMmKq27OXQ8+vtf1p5z47HzkLacmSd736enc/91te13/0y95bj543k8lSX7jk+vyyPe273X90le8IL/5xpOTJL/68bXZ9rT/lje+8oVZ/rMvT5K87a+u2+e//82vPiHvfP1LM7d9Phd+Yu0+1//SGSflrWe+KH9wzW37XaePXr0h5y+ZyNbeXH7rMzfu8+f/+8+8LG865cdz54Oz+Z09YXqq95xzct5w8guyYet383tX37rP9R84bzJnvOR5WXfvw/nDazbtc/2H33JKTj3xOfnaHQ/lY2vu2Of637/gtLx88aJ86dbv5K+/etc+1//J207PiceP5+qbtubyb9y7z/V/8Y4z8rxjF+az19+XK9dt2ef6y951VsYXjuWT192TL9x8f5JkenMv2+d37XW7uR3zuXT1JlHmR2IoUa61zic5vZRyfJLPlVJeVWu95am3KaUsT7I8SRYvXpypqalh3PVhbXZ21jodxExvbr+Xb5/flV6vl53fK99fv833bk+vt3eYjtr279dv2fJEeo/u/WS8dcejmZp6ePevtz6R3uN7X79l16OZmnowSfKd72xLb9veLxBtzmOZmvp/SZIHH9qW2e17X3/33Y9lamr3S6MPP7wt2+f3vv7OO2czteu+JElvP/+tt98xm6kn7skT8zW93rZ9rr/tttlMzd65z18WnvTI93Zkamoq/za3K73eE/tcv/6W9Tn6gY25f3b/1990003ZOTOWex+dT6+3fZ/rb7hhOo/dPZY7Htn/9ddff30efPZYNjy0/+vXfnNt7lt0VNY/sDO93r770K+77ro8f/yo3Hr//q//+te/nuMWlty2Zcd+t8xd+9Vr86yxkts3//v1Tw/yk2Z6c/5fPADPU8NVdr/6PMQDlvKRJI/XWv/oQLeZnJysmzbt+zdn9jY1NZVly5aNeoxmLb1kzX7DPHH8eL7+oXNGMFGbrFP/rNUPzvNUf0op62qtZx7qdsP49PXiPWfIKaWMJ3lTktsGPS4cykXnTmZ8wdhel40vGMtF506OaKI2Waf+WStGbRgvX5+Q5O/2vK98VJIraq1fGMJx4aCefI/v0tWbMtOby8Tx47no3Env/T2NdeqftWLUhvHp65uTLBnCLPADO3/JRM5fMuEltEOwTv2zVoySb/QCgEaIMgA0QpQBoBGiDACNEGUAaIQoA0AjRBkAGiHKANAIUQaARogyADRClAGgEaIMAI0QZQBohCgDQCNEGQAaIcoA0AhRBoBGiDIANEKUAaARogwAjRBlAGiEKANAI0QZABohygDQCFEGgEaIMgA0QpQBoBGiDACNEGUAaIQoA0AjRBkAGiHKANAIUQaARogyADRClAGgEaIMAI0QZQBohCgDQCNEGQAaIcoA0AhRBoBGiDIANEKUAaARogwAjRBlAGiEKANAI0QZABohygDQCFEGgEaIMgA0QpQBoBGiDACNEGUAaIQoA0AjRBkAGiHKANAIUQaARogyADRClAGgEaIMAI0QZQBohCgDQCNEGQAaIcoA0IihRbmUMlZKmS6lfGFYxwSAI8kwz5Tfm2TjEI8HAEeUoUS5lHJSkp9L8jfDOB4AHIlKrXXwg5RyZZKLkxyX5H/VWt+8n9ssT7I8SRYvXnzGFVdcMfD9Hu5mZ2ezaNGiUY/RCdaqP9apf9aqP9apP2efffa6WuuZh7rd0YPeUSnlzUkeqLWuK6UsO9Dtaq0rk6xMksnJybps2QFvyh5TU1OxTv2xVv2xTv2zVv2xTsM1jJevlyb5+VLKPUk+neScUsrlQzguABxRBo5yrXVFrfWkWutLk7w9yZpa6zsGngwAjjD2KQNAIwZ+T/mpaq1TSaaGeUwAOFI4UwaARogyADRClAGgEaIMAI0QZQBohCgDQCNEGQAaIcoA0AhRBoBGiDIANEKUAaARogwAjRBlAGiEKANAI0QZABohygDQCFEGgEaIMgA0QpQBoBGiDACNEGUAaIQoA0AjRBkAGiHKANAIUQaARogyADRClAGgEaIMAI0QZQBohCgDQCNEGQAaIcoA0AhRBoBGiDIANEKUAaARogwAjRBlAGiEKANAI0QZABohygDQCFEGgEaIMgA0QpQBoBGiDACNEGUAaIQoA0AjRBkAGiHKANAIUQaARogyADRClAGgEaIMAI0QZQBohCgDQCNEGQAaIcoA0AhRBoBGiDIANEKUAaARogwAjRBlAGiEKANAI0QZABpx9DAOUkq5J8ljSeaT7Ky1njmM4wLAkWQoUd7j7FrrQ0M8HgAcUbx8DQCNGFaUa5L/W0pZV0pZPqRjAsARZVgvXy+ttW4tpbwwyRdLKbfVWq996g32xPrJYD9RSrllSPd9OHtBEm8J9Mda9cc69c9a9cc69WeynxuVWutQ77WU8r+TzNZa/+ggt7neh8EOzTr1z1r1xzr1z1r1xzr1p991Gvjl61LKsaWU4578dZL/lMRZMAD8gIbx8vWPJ/lcKeXJ4/1DrfWaIRwXAI4oA0e51npXktf8gH9s5aD3e4SwTv2zVv2xTv2zVv2xTv3pa52G/p4yAPDM2KcMAI0YWZRLKW8tpWwopewqpfjk3tOUUs4rpWwqpXy7lPKhUc/TqlLKx0spD9hid3CllBeVUr5SStm45/+79456phaVUo4ppawtpdy0Z50+OuqZWlZKGSulTJdSvjDqWVpWSrmnlLK+lHJjKeX6g912lGfKtyS5IMm1h7rhkaaUMpbkz5P85ySnJPnlUsopo52qWZclOW/UQ3TAziTvr7W+MsnrkvxPj6n9eiLJObXW1yQ5Pcl5pZTXjXimlr03ycZRD9ERZ9daTz/UtqiRRbnWurHWumlU99+4s5J8u9Z6V611e5JPJ/mFEc/UpD1fUvPwqOdoXa31/lrrDXt+/Vh2P5FOjHaq9tTdZvf8dsGef3zwZj9KKScl+bkkfzPqWQ4n3lNu00SS+57y+y3xBMqQlFJemmRJkm+OdpI27XlJ9sYkDyT5Yq3VOu3fnyb5QJJdox6kA/r+Kuph/pSofZRSvpTkJ/Zz1e/WWv/ph3nfHVf2c5m/rTOwUsqiJP+Y5H211kdHPU+Laq3zSU4vpRyf3d/B8Kpaq88sPEUp5c1JHqi1riulLBv1PB1wyK+iftIPNcq11jf9MI9/GNuS5EVP+f1JSbaOaBYOE6WUBdkd5L+vtV416nlaV2vtlVKmsvszC6K8t6VJfr6U8l+SHJPk2aWUy2ut7xjxXE2qtW7d8+8HSimfy+63KPcbZS9ft+lbSU4upfxkKWVhkrcn+fyIZ6LDyu6v3PvbJBtrrX886nlaVUpZvOcMOaWU8SRvSnLbaKdqT611Ra31pFrrS7P7+WmNIO/fD/pV1KPcEvWLpZQtSV6f5J9LKatHNUtraq07k7w7yers/kDOFbXWDaOdqk2llE8luS7JZCllSynl10Y9U6OWJnlnknP2bMu4cc9ZDns7IclXSik3Z/dfjr9Ya7Xdh0H8eJKvlVJuSrI2yT8f7KuofaMXADTCy9cA0AhRBoBGiDIANEKUAaARogwAjRBlAGiEKANAI0QZABrx/wF4kbE2bfmv5gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "grid2d.state = coord2idx([3,3], 5)\n",
    "traj = grid2d.Episode()\n",
    "print(traj)\n",
    "\n",
    "grid2d.PlotTrajectory(traj)\n",
    "plt.gca().invert_yaxis()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 544,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Action 0\n",
      "[[-1.000e-01  0.000e+00 -1.000e-01 -9.000e-02  4.100e-02]\n",
      " [ 1.400e-02  9.657e+00 -7.000e-01  0.000e+00  7.680e-01]\n",
      " [ 0.000e+00  7.682e+00 -1.330e+00  0.000e+00  8.800e-02]\n",
      " [ 0.000e+00  4.558e+00 -3.652e+00 -2.407e+00  3.000e-03]\n",
      " [ 3.000e-03  2.092e+00  1.470e-01  3.000e-02  0.000e+00]]\n",
      "Action 1\n",
      "[[ 0.     0.     0.     0.     0.   ]\n",
      " [ 2.946 -1.897 -1.33   0.034 -0.26 ]\n",
      " [ 2.821 -3.987  0.    -0.7   -0.19 ]\n",
      " [ 0.     0.324  0.029  0.    -0.1  ]\n",
      " [ 0.879  0.021  0.     0.    -0.344]]\n",
      "Action 2\n",
      "[[ 0.481  0.     0.     0.019  0.   ]\n",
      " [ 0.418  2.233  0.    -0.7    0.   ]\n",
      " [ 0.     0.388 -1.33   0.     0.   ]\n",
      " [ 0.169  0.294  0.008  0.     0.   ]\n",
      " [-0.163  0.064 -0.1   -0.19  -0.1  ]]\n",
      "Action 3\n",
      "[[-0.158  0.     1.9    0.     2.855]\n",
      " [ 0.266  0.908 -0.7   -1.897  0.   ]\n",
      " [ 0.     0.078  0.    -0.7   -1.33 ]\n",
      " [-0.271  0.01   1.734  0.304  0.014]\n",
      " [-0.308  0.141  0.324  0.003  0.   ]]\n"
     ]
    }
   ],
   "source": [
    "grid2d.ShowFunction(grid2d.Q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 545,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[24, 19, 18, 17, 16, 11, 6, 1]\n",
      "[[0.         0.         0.         0.         0.        ]\n",
      " [0.         0.81       0.         0.         0.        ]\n",
      " [0.         0.6561     0.         0.         0.        ]\n",
      " [0.         0.531441   0.43046721 0.34867844 0.28242954]\n",
      " [0.         0.         0.         0.         0.22876792]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeUAAAHWCAYAAABJ3pFhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAcAklEQVR4nO3df7DcdX3v8deHQwKngEQ0Wjj4E7mnoigpDNWJ7UT0XuitWmTqaDs4xXaM7Vyttg7atDPa9o9CS6c69TrW2Focaeuvxii0Ta6SngEtSAlHiEkIv3/khGugcIRgQn597h8Jvcb8Ws+u7ud78njMOCa7m+++5zPLeZ7v7n52S601AMDwHTXsAQCAPUQZABohygDQCFEGgEaIMgA0QpQBoBF9RbmU8pZSytpSyu5SyjmDGgoAjkT9nil/J8lFSa4bwCwAcEQ7up9/XGtdnySllMFMAwBHMK8pA0AjDnumXEr5epKfPsBVf1hr/Uqvd1RKWZxkcZIce+yxZz//+c/vecgj1e7du3PUUX5v6oW16o116p216o116s0dd9zxSK11/uFud9go11pfP4iBaq1LkyxNkvHx8bphw4ZBHHZWm5iYyKJFi4Y9RidYq95Yp95Zq95Yp96UUu7v5XZ+vQGARvS7JerNpZSNSV6d5J9LKSsHMxYAHHn6fff1l5N8eUCzAMARzdPXANAIUQaARogyADRClAGgEaIMAI0QZQBohCgDQCNEGQAaIcoA0AhRBoBGiDIANEKUAaARogwAjRBlAGiEKANAI0QZABohygDQCFEGgEaIMgA0QpQBoBGiDACNEGUAaIQoA0AjRBkAGiHKANAIUQaARogyADRClAGgEaIMAI0QZQBohCgDQCNEGQAaIcoA0AhRBoBGiDIANEKUAaARogwAjRBlAGiEKANAI0QZABohygDQCFEGgEaIMgA0QpQBoBGiDACNEGUAaIQoA0AjRBkAGiHKANAIUQaARogyADRClAGgEaIMAI0QZQBohCgDQCNEGQAaIcoA0AhRBoBGiDIANEKUAaARogwAjRBlAGiEKANAI0QZABoxkCiXUi4opWwopdxVSvn9QRwTerF8cioLL1+VS1Y8mYWXr8ryyalhjwQwY0f3e4BSykiSjyf570k2JvmPUspXa63r+j02HMryyaksWbYmW3fsSpJMTW/NkmVrkiQXLhgb5mgAMzKIM+Vzk9xVa72n1ro9yeeS/PIAjguHdMXKDf8V5Kdt3bErV6zcMKSJAPrT95lykrEkD/7A3zcm+bkfvlEpZXGSxUkyf/78TExMDOCuZ7ctW7ZYp0OYmt560Mut24F5TPXOWvXGOg3WIKJcDnBZ3e+CWpcmWZok4+PjddGiRQO469ltYmIi1ungxm5cdcAwj80btW4H4THVO2vVG+s0WIN4+npjkuf9wN9PTbJpAMeFQ7r0/PGMzhnZ57LROSO59PzxIU0E0J9BRPk/kpxeSnlRKWVukrcl+eoAjguHdOGCsVx20ZmZO7LnYTw2bzSXXXSmN3kBndX309e11p2llHcnWZlkJMmna61r+54MenDhgrGceeqJuelbN+VX33DesMcB6MsgXlNOrfVfkvzLII4FP6rT5h+fB4/3OThA9/lJRud9fd13M7l557DHAOibKNN5n7r+nqy4d8ewxwDomygDQCNEGQAaIcoA0AhRBoBGiDKd95G3npXFrzhm2GMA9E2U6bxT5o3mWaMeykD3+UlG511966Z86yH7lIHuE2U676ob78+qB+xTBrpPlAGgEaIMAI0QZQBohCgDQCNEmc77xMVn590Ljh32GAB9E2U676Tj5uaEuWXYYwD0TZTpvC/e/GCu32hLFNB9Rw97AOjXl1ZvzPS0Dw8Bus+ZMgA0QpQBoBGiDACNEGUAaIQo03lXvuPc/N459ikD3SfKdN7o3JEcM2KfMtB9okznffaG+3Ktr24EZgH7lOm8a257yD5lYFZwpgwAjRBlAGiEKANAI0QZABohynTe59/16iz5udFhjwHQN1EGgEaIMp239Lq786/32qcMdJ99ynTetes326cMzArOlAGgEaIMAI0QZQBohCjTecfOGclc3xIFzAKiTOd95jfOzft9nzIwC4gyADRClOm8v7r2znzlru3DHgOgb/Yp03nfvOuRTE/vGvYYAH1zpgwAjRBlAGiEKANAI7ymTOc986fmZuf37VMGus+ZMp33128/O+9ZYJ8y0H2iDACNEGU6789W3J4vbrBPGeg+rynTebfc/5h9ysCs4EwZABohygDQCFEGgEZ4TZnOO/nEY3PUNvuUge5zpkznffRtC/KuV9qnDHSfKANAI0SZzvvjq9fm79c/NewxAPrmNWU6b92mxzP9+O5hjwHQN2fKANAIUQaARogyADRClOm8F88/Lj99nIcy0H3e6EXnXXbRKzIx8eiwxwDo20BOL0opny6lbC6lfGcQxwOAI9GgnvO7MskFAzoW9Gz55FTO+NCKXLLiySy8fFWWT04NeySAGRtIlGut1yXx/CE/Ucsnp7Jk2Zp8f/ue71Kemt6aJcvWCDPQWd4dQ2ddsXJDtu7Ytc9lW3fsyhUrNwxpIoD+/MTe6FVKWZxkcZLMnz8/ExMTP6m77qwtW7ZYp0OYmt560Mut24F5TPXOWvXGOg3WTyzKtdalSZYmyfj4eF20aNFP6q47a2JiItbp4MZuXHXAMI/NG7VuB+Ex1Ttr1RvrNFievqazLj1/PKNzRva5bHTOSC49f3xIEwH0Z1Bbov4xyQ1JxkspG0spvzmI48KhXLhgLJdddGbG5o0m2XOGfNlFZ+bCBWNDngxgZgby9HWt9VcHcRz4UV24YCwXLhjzFBowK3j6ms573+cm88lbtw17DIC++ZhNOu+h723L9LY67DEA+uZMGQAaIcoA0AhRBoBGeE2ZzvvZFzwzD+SJYY8B0DdnynTeBy/4mbxlfO6wxwDomygDQCNEmc77rc+uzscm7VMGuk+U6bzHvr89W7bbpwx0nygDQCNEGQAaIcoA0AhRpvMWvuTZOeNZI4e/IUDjRJnO+53XnZ5ffol9ykD3iTIANMLHbNJ5v/7pm/Loo9uyaNGwJwHojzNlOm/bjl3Zvss+ZaD7RBkAGiHKANAIUQaARogynfe6lz4nZz3HexaB7vOTjM5b/AunZWL3g8MeA6BvzpQBoBHOlOm8t37yhkxPb7VPGeg8Z8oA0AhRBoBGiDIANEKUAaARokznveEVJ+fck71nEeg+Uabz3v7qF+Z1z58z7DEA+ibKdN7W7bvylG+JAmYBUabzLvm7m/KXN28b9hgAfRNlAGiEKANAI0QZABohygDQCFGm837l7FPzmjH7lIHuE2U67y3nPC8/f6p9ykD3iTKd9+iT2/PEdvuUge4TZTrvt69anf89aZ8y0H2iDACNEGUAaIQoA0AjRBkAGmFzJ5138atekHXrnhz2GAB9E2U6742vPCUnPHbHsMcA6Junr+m8TdNb859bdw97DIC+iTKd97uf/3aW3vbUsMcA6JsoA0AjRBkAGiHKANAIUQaARogynffOn39xLniRr24Eus8+ZTrv9Wc8N0dvXj/sMQD65kyZzrv74S15aIt9ykD3iTKd9wfL1uTKtfYpA90nygDQCFEGgEaIMgA0QpQBoBGiTOe957zT86bT5g57DIC+2adM573m9Gdn59TIsMcA6JszZTpv7abv5f7Hdw17DIC+9R3lUsrzSin/VkpZX0pZW0p57yAGg14sn5zKmz/+7/nwv2/LwstXZfnk1LBHApixQTx9vTPJ+2utt5RSTkiyupTytVrrugEcGw5q+eRUlixbk+279nya19T01ixZtiZJcuGCsWGOBjAjfZ8p11ofqrXesvfPTyRZn8RPRH7srli5IVt37Pu09dYdu3LFyg1DmgigP6XWOriDlfLCJNcleXmt9fEfum5xksVJMn/+/LO/8IUvDOx+Z6stW7bk+OOPH/YYzbpkxZMHve7KC477CU7SHR5TvbNWvbFOvXnta1+7utZ6zuFuN7B3X5dSjk/yT0ne98NBTpJa69IkS5NkfHy8Llq0aFB3PWtNTEzEOh3c2I2rMjW9df/L541at4PwmOqdteqNdRqsgbz7upQyJ3uC/Pe11mWDOCYczqXnj2d0zr5boUbnjOTS88eHNBFAf/o+Uy6llCR/m2R9rfUv+x8JevP0m7muWLkhU9NbMzZvNJeeP+5NXkBnDeLp64VJ3p5kTSnl23sv+4Na678M4NhwSBcuGMvzThrNLbdM5p1vPm/Y4wD0pe8o11q/kaQMYBaYkT9fsSHT09vzzmEPAtAnn+gFAI0QZQBohCgDQCNEGQAaIcp03ofeeEZ+7aW+TxnoPt+nTOe97JQT8/AzfJ8y0H3OlOm8b9z5SNY+4vuUge4TZTrvY6vuzFfv3j7sMQD6JsoA0AhRBoBGiDIANEKUAaARokzn/elFZ+aSlx0z7DEA+ibKdN5p84/Pycd7KAPd5ycZnff1dd/N5Oadwx4DoG+iTOd96vp7suLeHcMeA6BvogwAjRBlAGiEKANAI0QZABohynTeR956Vha/wj5loPtEmc47Zd5onjXqoQx0n59kdN7Vt27Ktx6yTxnoPlGm86668f6sesA+ZaD7RBkAGiHKANAIUQaARogyADRClOm8T1x8dt694NhhjwHQN1Gm8046bm5OmFuGPQZA30SZzvvizQ/m+o22RAHdd/SwB4B+fWn1xkxP+/AQoPucKQNAI0QZABohygDQCFEGgEaIMp135TvOze+dY58y0H2iTOeNzh3JMSP2KQPdJ8p03mdvuC/X+upGYBawT5nOu+a2h+xTBmYFZ8oA0AhRBoBGiDIANEKUAaARokznff5dr86Snxsd9hgAfRNlAGiEKNN5S6+7O/96r33KQPfZp0znXbt+s33KwKzgTBkAGiHKANAIUQaARogynXfsnJHM9S1RwCwgynTeZ37j3Lzf9ykDs4AoA0AjRJnO+6tr78xX7to+7DEA+mafMp33zbseyfT0rmGPAdA3Z8oA0AhRBoBGiDIANMJrynTeM39qbnZ+3z5loPucKdN5f/32s/OeBfYpA90nygDQCFGm8/5sxe354gb7lIHu6/s15VLKsUmuS3LM3uN9qdb64X6PC7265f7H7FMGZoVBnCk/leS8Wusrk5yV5IJSyqsGcFw4rOWTU5l8YDobHtudhZevyvLJqWGP1KTlk1NZePmqXLLiSet0GNaqN9bpx6PvM+Vaa02yZe9f5+z9X+33uHA4yyensmTZmmzftTtJMjW9NUuWrUmSXLhgbJijNeXpddq6Y8+zCdbp4KxVb6zTj0/Z09Q+D1LKSJLVSV6S5OO11g8e6vbj4+N1w4YNfd/vbDcxMZFFixYNe4xmLbx8Vaamt+53+XHHjOTlp5y4z2Uvnn9cLrvoFUmSJctuyz0PP7nP9Wec8ox8+I0vS5K873OTeeh72/a5/mdf8Mx88IKfSZL81mdX57Hv7/sa9sKXPDu/87rTkyS//umbsm3Hvk+nv+6lz8niXzgtSfLWT96w38xveMXJefurX5it23flkr+7ab/rf+XsU/OWc56XR5/cnt++avV+11/8qhfkja88JZumt+Z3P//tfa6bfGD6v35x+UFzR47KgufPS5K857zT85rTn521m76XP7l63X63/cAF4zn7BSdl9f2P5s9X7P/f7ofeeEZedsqJ+cadj+Rjq+7c7/o/vejMnDb/+Hx93Xfzqevv2e/6j7z1rJwybzRX37opV914/37Xf+Lis3PScXPzxZsfzJdWb9zv+ivfcW5G547kszfcl2tue2i/6z//rlcnSZZed3euXb95n+uOnTOSz/zGuUmSM/9oZZ7YtnO/fz82bzTf/P3z8mcrbs8t9z+2z3Unn3hsPvq2BUmSP756bdZtenyf62fjY+9gj6mn14n9lVJW11rPOdztBrJPuda6K8lZpZR5Sb5cSnl5rfU7PzTQ4iSLk2T+/PmZmJgYxF3Palu2bLFOh3CgICfJk0/tyvT09D6XbdrxeCYmHt3z501PZfrJfX+gbNz9eCYmHk6SfPe72zK9bd9fVh/IE5mY+L9Jkocf2ZYt2/e9/t57n8jExJ6n7x59dFu279r3+rvv3pKJ3Q8mSaYPMPcdd27JxFP35aldNdPT2/a7/vbbt2Riy915YvuBr1+37smc8Ngd+c+tuzM9/dQ+1x3oh+fTlz+9Trfeemt2To3k/sd3ZXp6/zfN3XLLZJ64dyR3Pnbg62+++eY8/IyRrH3kwNff9K2b8uDxR2XN5p2Znt6x3/U33HBDnjV6VNY9dODrv/nNb+aEuSW3b9yR6en9o3nd9dflmJGSOx448PVP/3d09737Xz93pPzX9QcKcrLnsTYxMZEH7t++3/sXjtr2///9xo1PZfrxfdd7Nj72DvaYenqdmLmBnCnvc8BSPpzkyVrrXxzsNs6Ue+NM+dAOdqbst/V9WafeWaveWKcfXa9nyn2/0auUMn/vGXJKKaNJXp/k9n6PC4dz6fnjGZ0zss9lo3NGcun540OaqE3WqXfWqjfW6cdnEE9fn5zkM3tfVz4qyRdqrdcM4LhwSE+/oeSKlRsyNb01Y/NGc+n5495o8kOsU++sVW+eXo8PfOm2bN+12zoN0MCfvu6Fp6974+nr3lmr3lin3lmrw3vrJ2/I9PR0Vn7wF4c9SvN+Yk9fAwCDIcoA0AhRBoBGiDIAM/LOn39xLnjRnGGPMasM5MNDADjyvP6M5+bozeuHPcas4kwZgBm5++EteWjLgT/di5kRZQBm5A+WrcmVa586/A3pmSgDQCNEGQAaIcoA0AhRBoBGiDIAM/Ke807Pm06bO+wxZhX7lAGYkdec/uzsnBo5/A3pmTNlAGZk7abv5f7Hdw17jFlFlAGYkT+5el3+Yf32YY8xq4gyADRClAGgEaIMAI0QZQBohCgDMCMfuGA8v/Lf7FMeJPuUAZiRs19wUp641z7lQXKmDMCMrL7/0dz5mH3KgyTKAMzIn6/YkC/dYZ/yIIkyADRClAGgEaIMAI0QZQBohCgDMCMfeuMZ+bWX2qc8SPYpAzAjLzvlxDz8DPuUB8mZMgAz8o07H8naR+xTHiRRBmBGPrbqznz1bvuUB0mUAaARogwAjRBlAGiEKANAI0QZgBn504vOzCUvO2bYY8wqogzAjJw2//icfLyMDJLVBGBGvr7uu5ncvHPYY8wqogzAjHzq+nuy4t4dwx5jVhFlAGiEKANAI0QZABohygDQCFEGYEY+8tazsvgV9ikPkigDMCOnzBvNs0ZlZJCsJgAzcvWtm/Kth+xTHiRRBmBGrrrx/qx6wD7lQRJlAGiEKANAI0QZABohygDQCFEGYEY+cfHZefeCY4c9xqwiygDMyEnHzc0Jc8uwx5hVRBmAGfnizQ/m+o22RA3S0cMeAIBu+tLqjZme9uEhg+RMGQAaIcoA0AhRBoBGiDIANEKUAZiRK99xbn7vHPuUB0mUAZiR0bkjOWbEPuVBEmUAZuSzN9yXa31140DZpwzAjFxz20P2KQ+YM2UAaMTAolxKGSmlTJZSrhnUMQFo0/LJqUw+MJ0Nj+3OwstXZfnk1LBHmhUGeab83iTrB3g8ABq0fHIqS5atyfZdu5MkU9Nbs2TZGmEegIFEuZRyapJfSvI3gzgeAO26YuWGbN2xa5/Ltu7YlStWbhjSRLPHoN7o9dEkH0hywsFuUEpZnGRxksyfPz8TExMDuuvZa8uWLdapR9aqN9apd9bq4Kamtx70cmvWn76jXEp5Q5LNtdbVpZRFB7tdrXVpkqVJMj4+XhctOuhN2WtiYiLWqTfWqjfWqXfW6uDGblx1wDCPzRu1Zn0axNPXC5O8qZRyX5LPJTmvlHLVAI4LQIMuPX88o3NG9rlsdM5ILj1/fEgTzR59R7nWuqTWemqt9YVJ3pZkVa314r4nA6BJFy4Yy2UXnZmxeaNJ9pwhX3bRmblwwdiQJ+s+Hx4CwI/swgVjuXDBmKf5B2ygUa61TiSZGOQxAeBI4RO9AKARogwAjRBlAGiEKANAI0QZABohygDQCFEGgEaIMgA0QpQBoBGiDACNEGUAaIQoA0AjRBkAGiHKANAIUQaARogyADRClAGgEaIMAI0QZQBohCgDQCNEGQAaIcoA0AhRBoBGiDIANEKUAaARogwAjRBlAGiEKANAI0QZABohygDQCFEGgEaIMgA0QpQBoBGiDACNEGUAaIQoA0AjRBkAGiHKANAIUQaARogyADRClAGgEaIMAI0QZQBohCgDQCNEGQAaIcoA0AhRBoBGiDIANEKUAaARogwAjRBlAGiEKANAI0QZABohygDQCFEGgEaIMgA0QpQBoBGiDACNEGUAaIQoA0AjRBkAGiHKANAIUQaARhw9iIOUUu5L8kSSXUl21lrPGcRxAeBIMpAo7/XaWusjAzweABxRPH0NAI0YVJRrkv9TSlldSlk8oGMCwBFlUE9fL6y1biqlPCfJ10opt9dar/vBG+yN9dPBfqqU8p0B3fds9uwkXhLojbXqjXXqnbXqjXXqzXgvNyq11oHeaynlj5JsqbX+xSFuc7M3gx2edeqdteqNdeqdteqNdepNr+vU99PXpZTjSiknPP3nJP8jibNgAPgRDeLp6+cm+XIp5enj/UOtdcUAjgsAR5S+o1xrvSfJK3/Ef7a03/s9Qlin3lmr3lin3lmr3lin3vS0TgN/TRkAmBn7lAGgEUOLcinlLaWUtaWU3aUU79z7IaWUC0opG0opd5VSfn/Y87SqlPLpUspmW+wOrZTyvFLKv5VS1u/97+69w56pRaWUY0spN5VSbt27Tn887JlaVkoZKaVMllKuGfYsLSul3FdKWVNK+XYp5eZD3XaYZ8rfSXJRkusOd8MjTSllJMnHk/xikjOS/Gop5YzhTtWsK5NcMOwhOmBnkvfXWl+a5FVJ/pfH1AE9leS8Wusrk5yV5IJSyquGPFPL3ptk/bCH6IjX1lrPOty2qKFFuda6vta6YVj337hzk9xVa72n1ro9yeeS/PKQZ2rS3g+peXTYc7Su1vpQrfWWvX9+Int+kI4Nd6r21D227P3rnL3/88abAyilnJrkl5L8zbBnmU28ptymsSQP/sDfN8YPUAaklPLCJAuSfGu4k7Rp71Oy306yOcnXaq3W6cA+muQDSXYPe5AO6PmjqAf5LVH7KaV8PclPH+CqP6y1fuXHed8dVw5wmd/W6Vsp5fgk/5TkfbXWx4c9T4tqrbuSnFVKmZc9n8Hw8lqr9yz8gFLKG5JsrrWuLqUsGvY8HXDYj6J+2o81yrXW1/84jz+LbUzyvB/4+6lJNg1pFmaJUsqc7Any39dalw17ntbVWqdLKRPZ854FUd7XwiRvKqX8zyTHJnlGKeWqWuvFQ56rSbXWTXv/f3Mp5cvZ8xLlAaPs6es2/UeS00spLyqlzE3ytiRfHfJMdFjZ85F7f5tkfa31L4c9T6tKKfP3niGnlDKa5PVJbh/uVO2ptS6ptZ5aa31h9vx8WiXIB/ajfhT1MLdEvbmUsjHJq5P8cyll5bBmaU2tdWeSdydZmT1vyPlCrXXtcKdqUynlH5PckGS8lLKxlPKbw56pUQuTvD3JeXu3ZXx771kO+zo5yb+VUm7Lnl+Ov1Zrtd2Hfjw3yTdKKbcmuSnJPx/qo6h9ohcANMLT1wDQCFEGgEaIMgA0QpQBoBGiDACNEGUAaIQoA0AjRBkAGvH/AAQ3/mE65DLNAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "grid2d.state = coord2idx([4,4], 5)\n",
    "traj = grid2d.SARSA_lambda(eps=0., alpha=0., trace='accumulating')\n",
    "print(traj)\n",
    "\n",
    "grid2d.PlotTrajectory(traj)\n",
    "plt.gca().invert_yaxis()\n",
    "\n",
    "print(np.reshape(np.sum(grid2d.E,axis=1),(5,5)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 546,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeUAAAHWCAYAAABJ3pFhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de3xc5X3n8e9vZqQZXWzZlgW2MY65KlwSbMA0flG6MksDJmlpssXbdAtZQuql22ZpUpom7uuVZtOt27jebEmblDohBChNMCmXpthxklYqBhzf8A3jC5hLbHyXPLIlS3N99g/JDrJke4hOdM5z9Hm/XrxizzM68/OTkb56znl+Z8w5JwAAEL5E2AUAAIA+hDIAABFBKAMAEBGEMgAAEUEoAwAQEYQyAAARMaxQNrPbzWyrmZXN7NqgigIAYDQa7kr5ZUkflfRcALUAADCqpYbzxc65bZJkZsFUAwDAKMY1ZQAAIuKsK2Uz+7GkSUMM/alz7plKX8jM5kuaL0mZTOaaadOmVVzkaFUul5VI8HtTJZiryjBPlWOuKsM8VWbnzp2HnXNNZ3veWUPZOXdTEAU555ZIWiJJzc3NbseOHUEcNtba2trU0tISdhleYK4qwzxVjrmqDPNUGTN7q5Ln8esNAAARMdyWqI+Y2R5JsyU9a2YrgikLAIDRZ7i7r5+S9FRAtQAAMKpx+hoAgIgglAEAiAhCGQCAiCCUAQCICEIZAICIIJQBAIgIQhkAgIgglAEAiAhCGQCAiCCUAQCICEIZAICIIJQBAIgIQhkAgIgglAEAiAhCGQCAiCCUAQCICEIZAICIIJQBAIgIQhkAgIgglAEAiAhCGQCAiCCUAQCICEIZAICIIJQBAIgIQhkAgIgglAEAiAhCGQCAiCCUAQCICEIZAICIIJQBAIgIQhkAgIgglAEAiAhCGQCAiCCUAQCICEIZAICIIJQBAIgIQhkAgIgglAEAiAhCGQCAiCCUAQCICEIZAICIIJQBAIgIQhkAgIgglAEAiAhCGQCAiCCUAQCICEIZAICIIJQBAIgIQhkAgIgglAEAiAhCGQCAiCCUAQCICEIZAICIIJQBAIgIQhkAgIgglAEAiAhCGQCAiCCUAQCICEIZAICIIJQBAIgIQhkAgIgglAEAiAhCGQCAiCCUAQCIiFQQBzGzWyTdLykp6ZvOub8K4rjA2Txy3wL1ts9UoXqCdj78hDKNG3Tn4oVhlwUAP5dhr5TNLCnpa5LmSrpc0sfM7PLhHhc4m0fuW6DuzhtUSDdKZiqkG9XdeYMeuW9B2KUBwM8liNPX10l6zTn3unMuL+m7km4L4LjAGfW2z1Q5mR7wWDmZVm/7zJAqAoDhCeL09XmSdr/j73sk/dKpTzKz+ZLmS1JTU5Pa2toCeOl46+rqYp7OoFA94bSPM29D4z1VOeaqMsxTsIIIZRviMTfoAeeWSFoiSc3Nza6lpSWAl463trY2MU+nt/PhJ/pOXZ+iKt+hlpbbQ6go+nhPVY65qgzzFKwgTl/vkXT+O/4+VdLeAI4LnFGmcYMSpdyAxxKlnDKNG0KqCACGJ4hQXivpEjO7wMyqJf2WpH8J4LjAGd25eKHqGlbKygXJOVXl2lXXsJLd1wC8NezT1865opn9gaQV6muJ+pZzbuuwKwMqcOfihfrG7R+Wc2XN/94ySZy2BuCvQPqUnXPLJC0L4ljAuzUxu0/ODdrGAADeCSSUgTDVpi8jlAHEAqEM73U0/gqhDCAWCGV4r6QhevAAwEOEMrx3tJpIBhAPhDK8d6RhqPvXAIB/CGV4r8wnkAKICUIZ3pv22uMqy0n6ZNilAMCwEMrw3tjuDnZfA4gFQhneq828X2VCGUAMEMrwXseE2ayUAcQCoQzvFcMuAAACQijDe0czjruHAIgFQhney46hTxlAPBDK8F5ZSTmWygBigFCG9y7e9nD/Ri/6lAH4jVCG9zL5bnZfA4gFQhneq629mj5lALFAKMN7HeNmsVIGEAuEMrxXFB1RAOKBUIb3OjNEMoB4IJThvU76lAHEBKEM75WUDLsEAAgEoQzvXbH5G5Jzku4OuxQAGBZCGd5LuAK7rwHEAqEM79XUXScyGUAcEMrw3pGGmayUAcQCoQzvFcIuAAACQijDe501YVcAAMEglOG9o/WSRK8yAP8RyvBe2SW5zyaAWCCU4b0rN3+9f6MXfcoA/JYIuwAAANCHlTK8l6m/nrPXAGKBUIb3smOvpE8ZQCwQyvBeQezzAhAPhDK8l60Tt9kEEAuEMrx3rFaiTxlAHBDK8F7JOTlOYAOIAUIZ3nvf5iX9G73uCbsUABgW+pQRA05s9QIQB6yU4b302Dls9AIQC4QyvNdZ30yfMoBYIJThvTynrgHEBKEM72XrjN3XAGKBUIb3uuhTBhAThDK8V1Cea8oAYoFQhveu2vRQfyh/KuxSAGBYCGV4jxPXAOKCUIb3Ug0fZJsXgFgglOG9Y3UXcE0ZQCwQyvBenoYoADFBKMN7HWOMD1QGEAuEMrx3PCOx3QtAHBDK8F5Bx7mmDCAWCGV476pNj/aH8qfDLgUAhoVQhvcSMi4pA4gFQhn+G/8hGakMIAYIZXjveM15XFMGEAuEMryXo0sZQEwQyvBex1j6lAHEA6EM7/WkJfqUAcQBoQzv5V0nN9oEEAuEMrw3Y/N3+zd6fTbsUgBgWBJBHMTMvmVmB83s5SCOB7wbSWdKOk5fA/BfIKEs6duSbgnoWEDFHrlvgd6e/iXtuXiRltz1hB65b0HYJUXOokWL1NraOuCx1tZWLVq0KKSKAJxOIKHsnHtOUkcQxwIq9ch9C9TdeYPKqYxkpkK6Ud2dNxDMp5g1a5bmzZt3MphbW1s1b948zZo1K+TKAJyKa8rwVm/7TJXT6QGPlZNp9bbPDKmiaJozZ46WLl2qefPmae7cuVq+fLmWLl2qOXPmhF0agFOMWCib2XxJ8yWpqalJbW1tI/XS3urq6mKezqBQPeG0jzNvA5mZ5s6dq0cffVR33HGHzIw5Ogu+/yrDPAVrxELZObdE0hJJam5udi0tLSP10t5qa2sT83R6Ox9+QoV046DHq/Idamm5PYSKoqu1tVXLly/XHXfcoeXLl+uuu+5ipXwWfP9VhnkKVlAbvYARl2ncoEQpN+CxRCmnTOOGkCqKphPXkJcuXapPfOITJ09ln7r5C0D4gmqJ+o6kVZKazWyPmd0dxHGBM7lz8ULVNaxUurddck5VuXbVNazUnYsXhl1apKxdu3bANeQT15jXrl0bcmUAThXI6Wvn3MeCOA7wbt25eKG+f9MsOef06/+2ThKnrU/12c8OvqnKnDlzOH0NRBC7r+G9fNPtfHQjgFgglOG9fPU4QhlALBDK8F6vHB8SBSAWCGV47/A4Pk8ZQDwQyvBevkpiqQwgDghleC9f2s81ZQCxQCjDezNefqY/lP8s7FIAYFgIZXiv2iVUZqUMIAYIZXiv+9zfZqMXgFgglOG9UqqOa8oAYoFQhvd65eTYfA0gBghleO/QeJNYKAOIAUIZ3ivwLgYQE/w4g/cKxd1cUwYQC4QyvHfV1mX9ofx/wi4FAIaFUIb30i4pVy6HXQYADBuhDO8dnfRxOXZ6AYgBQhnec8kqrikDiAVCGd7r4fOUAcQEoQzvHZpg3GUTQCwQyvBeMRl2BQAQDEIZ3isW3pBz7L4G4D9CGd57/ys/7N/o9eWwSwGAYSGU4b2aclJlVsoAYoBQhvc6zvskLVEAYoFQhveMdigAMUEow3v0KQOIC0IZ3jvYSJ8ygHgglOG9UiLsCgAgGIQyvFfK7aRPGUAsEMrw3vu2t7L7GkAsEMrwXo3L0KcMIBYIZXivY8rHWSkDiAVCGd6jTxlAXBDK8N5x+pQBxAShDO8dmGgSZ68BxAChDO85EytlALFAKMN7rudllcvsvgbgP0IZ3rti5wvsvgYQC4QyvFejMXJipQzAf4QyvNcx+bdZKQOIBUIZ3qNPGUBcEMrwHn3KAOKCUIb39jeRyADigU+iBQAgIlgpw3uJrg0ql9noBcB/hDK8995da9h9DSAWCGV4ryYxXo6VMoAYIJThvY5z57FSBhALhDL8x+ZrADFBKMN7x40+ZQDxQCjDewcmksgA4oE+ZQAAIoJQhvdSnWuUPLI67DIAYNg4fQ3vXfrmBnZfA4gFQhneq0mcQygDiAVCGd7rOPcjciKUAfiPa8rwH5uvAcQEK2V4rzvBKhlAPBDK8N7BRpbKAOKB09eIAeOKMoBYIJThvUzHSmUOrwy7DAAYNk5fw3sX7t5KSxSAWCCU4b3a1BSVy2FXAQDDN+zT12Z2vpm1mtk2M9tqZvcGURhQiUfuW6BdU/9Ar0//Qy256wk9ct+CsEuKnEWLFqm1tXXAY62trVq0aFFIFUUXc4WwBXFNuSjpj5xzl0n6gKTfN7PLAzgucEaP3LdA3Z03yCWrJDMV0o3q7ryBYD7FrFmzNG/evJNh09raqnnz5mnWrFkhVxY9zBXCNuzT1865fZL29f/5mJltk3SepFeGe2zgTHrbZ6qcTg94rJxMq7d9ZkgVRdOcOXO0dOlSzZs3T3PnztXy5cu1dOlSzZkzJ+zSIoe5QtgCvaZsZtMlzZQ06CN7zGy+pPmS1NTUpLa2tiBfOpa6urqYpzMoVE847ePM20Bmprlz5+rRRx/VHXfcITNjjk6DuXp3+DkVLAtq16qZ1Uv6D0l/4Zx78kzPbW5udjt27AjkdeOsra1NLS0tYZcRWUvuekKFdOOgx6ty7Zr/0O0hVBRdJ07Dsvo7O+bq3eHnVGXMbL1z7tqzPS+QPmUzq5L0z5IeO1sgA0HJNG5QopQb8FiilFOmcUNIFUXTiZBZunSpPvGJT5w8PXvqhiYwVwhfELuvTdKDkrY5574y/JKAyty5eKHqGlYq3dsuOaeqXLvqGlbqzsULwy4tUtauXTtgtXfiuunatWtDrix6mCuEbdinr83slyWtlLRF0olu0QXOuWWn+xpOX1eG00KVWTH3v8iprFuWPxV2KZHHe6pyzFVlmKfKVHr6Oojd18+LD89DiNqbbpa4+zWAGODe1/Ces5+dogEAn3GbTXivK+VYJwOIBUIZ3js8nqsnAOKBUIb3HJ+nDCAmCGV4b8Le5XJlJ+l3wy4FAIaFUIb3Jh96i89TBhALhDK8V1N9iUQoA4gBQhne62i8UfQpA4gDQhneK5vIZACxQCjDe11V9CkDiAdCGd5rH0efMoB4IJThvbIS4vw1gDgglOG9SW89LefKok8ZgO8IZXhvYnYffcoAYoFQhvdq05cRygBigVCG9zoaf4VQBhALhDK8VxLbvADEA6EM7x2tJpIBxAOhDO8daaBPGUA8EMrwXl+fMgD4j1CG96a99rjKcpI+GXYpADAshDK8N7a7g93XAGKBUIb3ajPvV5lQBhADhDK81zFhNitlALFAKMN7xbALAICAEMrw3tGM4+4hAGKBUIb3smPoUwYQD4QyvFdWUo6lMoAYIJThvYu3Pdy/0Ys+ZQB+I5ThvUy+m93XAGKBUIb3amuvpk8ZQCwQyvBex7hZrJQBxAKhDO8VRUcUgHgglOG9zgyRDCAeCGV4r5M+ZQAxQSjDeyUlwy4BAAJBKMN7V2z+huScpLvDLgUAhoVQhvcSrsDuawCxQCjDezV114lMBhAHhDK8d6RhJitlALFAKMN7hbALAICAEMrwXmdN2BUAQDAIZXjvaL0k0asMwH+EMrxXdknuswkgFghleO/KzV/v3+hFnzIAvyXCLgAAAPRhpQzvZeqv5+w1gFgglOG97Ngr6VMGEAuEMrxXEPu8AMQDoQzvZevEbTYBxAKhDO8dq5XoUwYQB4QyvFdyTo4T2ABigFCG9963eUn/Rq97wi4FAIaFPmXEgBNbvQDEAStleC89dg4bvQDEAqEM73XWN9OnDCAWCGV4L8+pawAxQSjDe9k6Y/c1gFgglOG9LvqUAcQEoQzvFZTnmjKAWCCU4b2rNj3UH8qfCrsUABgWQhne48Q1gLgglOG9VMMH2eYFIBaGHcpmlpH0nKR0//G+55z7s+EeF6jUsboLuKYMIBaCWCnnJN3onOsysypJz5vZcufcTwI4NnBGj9y3QF3V/0kukdKSu55QpnGD7ly8MOyyImfjA8u0fm2velMN2vXok7pmVkYz7rk17LIiiblCmIZ972vXp6v/r1X9/7FswS/cI/ctUHfnDXLJKslMhXSjujtv0CP3LQi7tEjZ+MAyrVqfUG/VOMlMvVXjtGp9QhsfWBZ2aZHDXCFsgVxTNrOkpPWSLpb0Nefc6iCOC5xJb/tMldPpAY+Vk2kdz16v7/zuP0mSqqZMVnJsg8akc7pk9d9LkjZXz1ZXokGSVD11qhL19RqX7tGFq/9BkrQhfYN6rK5vfNo0JWpr1Zg+pvesflCStC7dorxl+sanT1cik9E51VlNXfNtSdLq9E0qWd+3VvrCC2XV1ZpUdUhT1j4mSXoxc8vJetMXXyxLpXRecp/OXf+4SkpqdeZXT45nLr1USiR0vr2lpg1PKa+01mXm/Gz8ve+VJE0v7dSELcvUY7XakP4VSZIlEkpfeqn270ipnBz4rV5OVuvFlwp64/++JEm6+Pg6jd35vDoTE7S1+rq+r6+uVvrCCyVJlx59XvWvr1NHoknbq6+RJCUyGVVPny5Jeu/hH6t2z8s6lJisV6uv6huvrVX1tGmSpMv2fl81B3fpQHKqdlVdKUlK1teraupUSdKVbzyu6s592pucrjer+v5NybFjVTVlSt/49m+puveodqcu1u7UxX3j48apatIkSdJVG/9WSZX0ZqpZe1MXSJJSEyYodc45Urmsqzd/VZK0K3WFDqTO7xufOFGpiROVtJKu2vC3/f/fzVM5lRk0V+vXZjWDDyHDCAgklJ1zJUkzzGycpKfM7Ern3MvvfI6ZzZc0X5KamprU1tYWxEvHWldXF/N0BoXqCUM+XkpmVMy3S5LyXd0ql516E91qymYlSblxeRVTxb4/dx2TKxaVT3RqQv94fnxexWRf2OeOHZPL51WydjX0jxcmFFRM9H3r5I4elevtldMB1fePFxuLKvVvCc8dPSqXSsnKB1R7Ynxi8WStvZ2dUjKpZHG/0tmsSpY6WZskZTs7JTNV5/arKptVIZEZON5/zP3HDyiRzSqXLKo4vn/cTD3ZrMqWHHKenKVOfv3B7EGVs1l1VVWp2PCz4/ecGD98SMVsVl3VNSqO7Rt3+ZyO948fOnRIDdmsutNjVEz0j+d+Nt5++LDqs1l1ZcarWN83XujtVfeJ8Y521RzLqrvmuIp1g8ezR46oOndMx2uPq1jbP97T87Pxzk4lXVE9dT0q1vSN548fVzmblZw7+e/sqe9RMTNwPKHSyfHilJoh56o31cD34mnwcypYFvQGGTP7M0ndzrnFp3tOc3Oz27FjR6CvG0dtbW1qaWkJu4zIWnLXEyqkGwc9XpVr1/yHbg+homh68O4n+07HniJTyOruBz8aQkXRxVy9e/ycqoyZrXfOXXu25w37mrKZNfWvkGVmNZJukrR9uMcFzibTuEGJUm7AY4lSTpnGDSFVFE3XzMooUcoPeCxRyuuaWZnTfMXoxVwhbMMOZUmTJbWa2WZJayX9yDn3rwEcFzijOxcvVF3DSlXl2iXnVJVrV13DSnZfn2LGPbdq9jVlZQp9p3IzhaxmX1NmR/EQTs5VbwdzhVAM+5qyc26zpJkB1AK8a3cuXig99CFls1mN+/QLkjhtPZQZ99yqGfecONXIadgzmXHPrRr/wp3K7s3qqu//S9jlYJThjl7w3t9YVr01vfpc2IUgNrZVXa3c2JyuCrsQjDqEMry3pnhc5UQ57DIQI0eS56hYXTz7E4GAEcrwXrHsxF02AcQBoQzvtSebVC6Vwi4DMVI1aZLy3d1hl4FRiFCG97oSY1Usc6oRwUmOGycuiCAMhDK8N7ZUrUIxiO4+oE9NVUHHrTfsMjAKEcrw3t+3v618oRB2GYiRy9Z+rf/Wm/QnY2SxvID3Lj5njKbU8VYG4D9WyvDel+2IcrU5fSHsQhAbW6uvU66BPmWMPEIZ3ttY6FHZ2JaD4HQmJqhYxeZBjDxCGd4rOfqUAcQDoQzvHU6eo1KJlTKCUzVlsvJd9Clj5BHK8F53Ygx9yghUcmyDymVOv2DkEcrwXmOxhpYoBGpMOqfeBCtljDxCGd77u+xu5XL5sz8RqNAlq/9eTdmspF8LuxSMMjR3wnsXTqzXZPqUAcQAK2V474vWrnxtXgvDLgSxsbl6tnLj8vQpY8QRyvDejkJOZT4+AAHqSjSomGLzIEYeoQzvlZwT+2QBxAGhDO8dTE5Sic9TRoCqp05VrutY2GVgFCKU4b2eRB19yghUor5ersh7CiOPUIb3JhVqVaBPGQEal+5RPtEZdhkYhQhleO/+o7uVy+XCLgMxcuHqf9CEbFbSR8IuBaMMzZ3w3vTGOp1by1sZgP9YKcN7n7PDytcV9JWwC0FsbEjfoPx4+pQx8ghleO/NQl7O0aeM4PRYnYrJdNhlYBQilOG9Mn3KAGKCUIb3DiSnqEifMgJUPW2acsfoU8bII5Thvd5EDX3KCFSitlYuzyePYeQRyvDee/J1yhf4AYrgNKaPqWTtYZeBUYhQhvcWd+1Wb6437DIQI+9Z/aAasllJt4ddCkYZmjvhvWkTanVODW9lAP5jpQzvfdoOqVBf0N+FXQhiY126RYUJBfqUMeIIZXhvX76gMn3KCFDeMiom+PGIkcc5P3ivLPqUAcQDvwrCe/tSU1XiY/YQoOrp05U7ejTsMjAKEcrwXt7SKloy7DIQI4lMRq6XHf0YeYQyvHdxrl4F+pQRoHOqs3I6EHYZGIUIZXjvyz0/VW8PqxoEZ+qab6s+m5X0sbBLwShDKMN7U8fVKitWygD8RyjDe/fYQRXrC/pm2IUgNlanb1KxsUifMkYcoQzvHckX5BxNUQhOyVIqWdhVYDSiTxnec/3/AYDvWCnDe3tT01SkTxkBSl94IX3KCAWhDO8VrEpF41wjgmPV1XIpfjxi5PGug/eu7K1Tng+kR4AmVR2SlelTxsgjlOG9P8/tUU9PT9hlIEamrH1MtdmspDvDLgWjDKEM701pqFHW5cIuAwCGjVCG9+6yAyqOKerRsAtBbLyYuUXFifQpY+QRyvBed75InzKAWKBPGQCAiGClDO/tSb2HPmUEKn3xxert7Ay7DIxChDK8xy0RETRLpaQkn9GNkUcow3vX9NCnjGCdl9ynZHF/2GVgFCKU4b0vFN5WT+/xsMtAjJy7/nGls1lJd4ddCkYZQhneaxhbLZW5eQiCU1JSJePHI0Ye7zp47/fskEr0KSNAqzO/qmKqqKvDLgSjDqEM7x2nTxlATNCnDABARLBShvd+mrpAJfqUEaDMpZcqS58yQsBKGd5zllDZeCsjQImExGd0IwSslOG964/X0qeMQJ1vb6k6t1/SjWGXglGGUIb3Plfaq+M5+pQRnKYNT6kqm5X0e2GXglGGUIb3qsZUSeWwq0Cc5JVWIZEJuwyMQhZUK4mZJSWtk/S2c+7DZ3puc3Oz27FjRyCvG0dPb3hbf71ih97O9ui8cTX645ub9Rszzwu7rEj66hOf0be7fqiCSecUnW5ruFn/6/avhF1W5Dz7+rO6/6X7ta97nybXTda9V9+rD134obDLiqSNDyzTiy8l5SylTLFT18zKaMY9t4ZdVuTsXL1fq57Zpa6OnOonpDX7tot06S9NCrusyDKz9c65a8/2vCB3x9wraVuAxxuVnt7wtj7/5Ba9ne27Q9Xb2R59/sktenrD2yFXFj1ffeIzerRrhQoJk8x0sCqhR7tW6KtPfCbs0iLl2def1Rdf/KL2de+TJO3r3qcvvvhFPfv6syFXFj0bH1imVesTcokqyUy9VeO0an1CGx9YFnZpkbJz9X61PrZdXR05SVJXR06tj23XztXcL3y4Ajl9bWZTJX1I0l9I4ifiMPz1ih3qKZQGPNZTKOmz39us76z5qT78/sm6Y/Z09eRL+u8PrRn09b95zVTdfu356ujO6/f+cf2g8d/5wHv0a1dN0d5sjz79+MZB4797w4W66fJztetQlxY8uWXQ+KduvES/fMlEbd3bqS99/5VB45+9pVnXvGeC1r/VoUU/GHw25Au/drmumNKg5189rL/991cHjS/86Pt0UVO9fvzKAX1j5euDxv/ff52hKeNq9P1Ne/VM5wr1Vg38vbI3kdAznSs0r3u/Pr/y84O+/uNXfFwt57fojc439KVVXxo0Pv/98zV7ymxt79iuL6/58qDxe6++VzPOmaGNBzfq/pfuHzT+J9f9id474b1atXeVlmxeMvjfP/sLuqDhArXtbtPDWx8eNP6XN/ylJtVN0g/e+IEe3/H4oPGvtHxF4zPj9fRrT+uZ154ZNP71m76umlSNvrv9u1rx5gpJ0uZDm5UvD9wI11vq1f0v3c9q+RTr1/aqXDVuwGPlZLXWr83qonm9+vFDg9/zM351mi54/0Qd2d+ttscGv+evvXW6zr9sgg7tPqbnlw5+z3/gNy7S5IsatG9Xp37y9K5B47887xI1nT9Gu7d1aN2yNweNt/y3Zo2fVKc3Nh/Wxh/9dND4TXddrjETMnp13QG9/B+Df7m/5X9cqZr6am17cZ+2r9o3aPzDn7pKVdVJbWnbo9fWH5QkHXijU6XiwLOsxXxZq57ZxWp5mIK6pvw3kj4raczpnmBm8yXNl6Smpia1tbUF9NLxcmKFfKp8qaxsNqudr3apLfemciWnbLZ30PO2b+9SW9cuHcsPPf7KK90ac2Sn2nvKymZzg8a3vLxFqYPbtK9r6PFNmzap+HZSbx0tKZsdvOP5pZc26NgbSb16ZOjxdevW6dDYpLYeHnp8zeo12l2f0JaDRWWzhUHjq1atUmNNQq/sK+pQauiWlUMp06pVq5TNZgf/+7ZskXZJBwoHhhzftGmTcjtz2pPfM+T4Sy+9pGwmq9d7Xx9yfN26ddpfvV/be7Yr2zl4fM2aNXqr6i1tOb5F2aODx1etWqXxqfHa2r1V2WODx1944QXVJ+u1vWu7sl2Dx1c+t1LViWrtPLZT2e6+8VMD+WZrYRkAAAb4SURBVIR93fv4PjxFb6rhtI//ZNUqZbODL/e9vKVTb3WYckfdkOObNm3SrgOmniNDj2/Y8JJ27DYdPzz0+Lp161Szy9S1f+jxNWvWKD3WdOztocd/smqVqupMnT8devyFF15QKm068vrQ4yufe06JlKnjVafO/vHSaW4L0NWR4z01TMO+pmxmH5Z0q3Puf5pZi6T7uKb887v+r/59yGA+b1yNXvgc7Rnv9J+/eYUOVg2+AnNOoax/++TWECqKpg9+74MnT12/0+S6yfrhb/4whIqi68G7n1TvKStlScoUsrr7wY+GUFE0PbzghZOnrt+pfkJaH194fQgVRd9IXlO+XtKvm9mbkr4r6UYz+8cAjjsq/fHNzaqpGvjh6jVVSf3xzc0hVRRdtzXcrEx54LbrTLms2xpuDqmiaLr36nuVSQ7cSZxJZnTv1feGVFF0XTMro0Rp4JmFRCmva2axE/udZt92kVLVA+MjVZ3Q7NsuCqmi+Ahs97UksVIOBruvK/fVJz6jZzpX6FDK1MTu69Ni93XlNj6wTOvX9qo31cDu6zNg9/W7U+lKmVCOsLa2NrW0tIRdhheYq8owT5VjrirDPFWm0lAO9OYhzrk2SW1BHhMAgNGCu/gDABARhDIAABFBKAMAEBGEMgAAEUEoAwAQEYQyAAARQSgDABARhDIAABFBKAMAEBGEMgAAEUEoAwAQEYQyAAARQSgDABARhDIAABFBKAMAEBGEMgAAEUEoAwAQEYQyAAARQSgDABARhDIAABFBKAMAEBGEMgAAEUEoAwAQEYQyAAARQSgDABARhDIAABFBKAMAEBGEMgAAEUEoAwAQEYQyAAARQSgDABARhDIAABFBKAMAEBGEMgAAEUEoAwAQEYQyAAARQSgDABARhDIAABFBKAMAEBGEMgAAEUEoAwAQEYQyAAARQSgDABARhDIAABFBKAMAEBGEMgAAEUEoAwAQEYQyAAARQSgDABARhDIAABFBKAMAEBGEMgAAEUEoAwAQEYQyAAARQSgDABARhDIAABFBKAMAEBGEMgAAEUEoAwAQEYQyAAARQSgDABARhDIAABFBKAMAEBGpIA5iZm9KOiapJKnonLs2iOMCADCaBBLK/eY45w4HeDwAAEYVTl8DABARQYWyk/RDM1tvZvMDOiYAAKNKUKevr3fO7TWzcyT9yMy2O+eee+cT+sP6RGDnzOzlgF47ziZK4pJAZZiryjBPlWOuKsM8Vaa5kieZcy7QVzWzL0rqcs4tPsNz1rEZ7OyYp8oxV5VhnirHXFWGeapMpfM07NPXZlZnZmNO/FnSByWxCgYA4F0K4vT1uZKeMrMTx/sn59wPAjguAACjyrBD2Tn3uqSr3uWXLRnu644SzFPlmKvKME+VY64qwzxVpqJ5CvyaMgAA+PnQpwwAQESEFspmdruZbTWzspmxc+8UZnaLme0ws9fM7HNh1xNVZvYtMztIi92Zmdn5ZtZqZtv6v+/uDbumKDKzjJmtMbNN/fP0v8OuKcrMLGlmG8zsX8OuJcrM7E0z22JmG81s3ZmeG+ZK+WVJH5X03NmeONqYWVLS1yTNlXS5pI+Z2eXhVhVZ35Z0S9hFeKAo6Y+cc5dJ+oCk3+c9NaScpBudc1dJmiHpFjP7QMg1Rdm9kraFXYQn5jjnZpytLSq0UHbObXPO7Qjr9SPuOkmvOeded87lJX1X0m0h1xRJ/Tep6Qi7jqhzzu1zzr3U/+dj6vtBel64VUWP69PV/9eq/v/YeDMEM5sq6UOSvhl2LXHCNeVoOk/S7nf8fY/4AYqAmNl0STMlrQ63kmjqPyW7UdJBST9yzjFPQ/sbSZ+VVA67EA9UfCvqID8lahAz+7GkSUMM/alz7plf5Gt7zoZ4jN/WMWxmVi/pnyX9oXPuaNj1RJFzriRphpmNU989GK50zrFn4R3M7MOSDjrn1ptZS9j1eOCst6I+4Rcays65m36Rx4+xPZLOf8ffp0raG1ItiAkzq1JfID/mnHsy7HqizjmXNbM29e1ZIJQHul7Sr5vZrZIyksaa2T86534n5LoiyTm3t/9/D5rZU+q7RDlkKHP6OprWSrrEzC4ws2pJvyXpX0KuCR6zvlvuPShpm3PuK2HXE1Vm1tS/QpaZ1Ui6SdL2cKuKHufc551zU51z09X38+nfCeShvdtbUYfZEvURM9sjabakZ81sRVi1RI1zrijpDyStUN+GnKXOua3hVhVNZvYdSaskNZvZHjO7O+yaIup6SXdIurG/LWNj/yoHA02W1Gpmm9X3y/GPnHO0+2A4zpX0vJltkrRG0rNnuhU1d/QCACAiOH0NAEBEEMoAAEQEoQwAQEQQygAARAShDABARBDKAABEBKEMAEBEEMoAAETE/wf1FKZw9C8SyAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "grid2d.state = 20\n",
    "traj = grid2d.Episode()\n",
    "fig = grid2d.PlotTrajectory(traj)\n",
    "\n",
    "for s in [21,22,23,24]:\n",
    "    grid2d.state = s\n",
    "    traj = grid2d.Episode()\n",
    "    #print(traj)\n",
    "\n",
    "    grid2d.PlotTrajectory(traj, fig=fig)\n",
    "\n",
    "for h in holes:\n",
    "    plt.plot(h[1],h[0],'kx')\n",
    "plt.gca().invert_yaxis()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}