vector_speedup.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Basic Reinforcement Learning\n",
    "Purpose: To learn about RL from its most basic instantiation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import time\n",
    "from copy import deepcopy\n",
    "\n",
    "dtype = torch.float32\n",
    "if torch.cuda.is_available():\n",
    "    device = torch.device(\"cuda:6\") # Uncomment this to run on GPU\n",
    "else:\n",
    "    device = torch.device(\"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cpu')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make('LunarLander-v2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make('CartPole-v0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'CartPoleEnv' object has no attribute 'clone'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-bf589e2cb706>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0menv2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclone\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/gym/core.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m    221\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'_'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    222\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"attempted to get missing private attribute '{}'\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 223\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    224\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    225\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'CartPoleEnv' object has no attribute 'clone'"
     ]
    }
   ],
   "source": [
    "env2 = env.clone()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Vector_env(gym.Wrapper):\n",
    "    '''\n",
    "     class Vector_env\n",
    "     \n",
    "     Acts as a wrapper for the OpenAI gym environments.\n",
    "    '''\n",
    "    def __init__(self, env, n=1):\n",
    "        '''\n",
    "         venv = Vector_env(env, n=1)\n",
    "         \n",
    "         Creates n copies of env.\n",
    "        '''\n",
    "        super().__init__(env)\n",
    "        self.envs = []\n",
    "        self.n_envs = n\n",
    "        for k in range(n):\n",
    "            env_r = deepcopy(env)\n",
    "            env_r.seed(np.random.randint(10000))\n",
    "            self.envs.append(env_r)\n",
    "    \n",
    "    def reset(self):\n",
    "        '''\n",
    "         states = venv.reset()\n",
    "         \n",
    "         Resets all the environments, and returns an array with the\n",
    "         reset states in each row.\n",
    "        '''\n",
    "        states = []\n",
    "        for e in self.envs:\n",
    "            state = e.reset()\n",
    "            states.append(state)\n",
    "        self.S = np.array(states)\n",
    "        return np.array(states)\n",
    "    \n",
    "    def step(self, actions):\n",
    "        '''\n",
    "         S, R, done, info = venv.step(actions)\n",
    "         \n",
    "         Takes one step in each of the environments, and returns an array\n",
    "         for each of the resulting outputs.\n",
    "         \n",
    "         Inputs:\n",
    "           actions   an array of actions\n",
    "           \n",
    "         Outputs:\n",
    "           S    array of states after the step\n",
    "           R    array of rewards from the step\n",
    "           done array of Boolean flags:\n",
    "                  False means episode continues\n",
    "                  True means episode is done\n",
    "        '''\n",
    "        S, R, dones = [], [], []\n",
    "        for e,a in zip(self.envs, actions):\n",
    "            next_s, r, done, inf = e.step(a)\n",
    "            if done:\n",
    "                S.append(e.reset())\n",
    "            else:\n",
    "                S.append(next_s)\n",
    "            R.append(r)\n",
    "            dones.append(done)\n",
    "        return np.array(S), np.array(R), np.array(dones)\n",
    "    \n",
    "    def state(self):\n",
    "        S = []\n",
    "        for e in self.envs:\n",
    "            S.append(e.state)\n",
    "        return np.array(S)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Agent():\n",
    "    '''\n",
    "     class Agent\n",
    "     \n",
    "     Represents an RL agent. Its Q function is represented by a neural network, so that\n",
    "       agent.Q(s) returns a vector of probabilities over action-space.\n",
    "    '''\n",
    "    def __init__(self, envs):\n",
    "        '''\n",
    "         ag = Agent(envs)\n",
    "         \n",
    "         Instantiates an Agent object.\n",
    "         \n",
    "         Inputs:\n",
    "           envs  a Vector_env object, containing the environments for the task\n",
    "        '''\n",
    "        self.envs = envs\n",
    "        self.state_dim = self.envs.observation_space.shape[0]  #state_dim\n",
    "        self.n_actions = self.envs.action_space.n  #n_actions\n",
    "        #self.state = torch.zeros(self.state_dim)\n",
    "        self._max_episode_steps = 100\n",
    "        \n",
    "        #**** DEBUG ****\n",
    "        #self.a_pointer = 0\n",
    "        #***************\n",
    "        \n",
    "        # Policy\n",
    "        self.hidden = 512\n",
    "        self.Pi_net = torch.nn.Sequential(\n",
    "                    torch.nn.Linear(self.state_dim, self.hidden),\n",
    "                    torch.nn.ReLU(),\n",
    "                    torch.nn.Linear(self.hidden, 256),\n",
    "                    torch.nn.ReLU(),\n",
    "                    torch.nn.Linear(256, self.n_actions),\n",
    "                    torch.nn.LogSoftmax(dim=-1)).to(device)\n",
    "        \n",
    "        \n",
    "        # Value function\n",
    "        self.V_hidden = 512\n",
    "        self.V_net = torch.nn.Sequential(\n",
    "                    torch.nn.Linear(self.state_dim, self.V_hidden),\n",
    "                    torch.nn.ReLU(),\n",
    "                    torch.nn.Linear(self.V_hidden, 256),\n",
    "                    torch.nn.ReLU(),\n",
    "                    torch.nn.Linear(256, 1)).to(device)\n",
    "        \n",
    "        # Eligibility-trace buffer for Pi_net\n",
    "        self.Piz = []\n",
    "        for p in self.Pi_net.parameters():\n",
    "            self.Piz.append(torch.zeros_like(p))\n",
    "            \n",
    "        # Eligibility-trace buffer for V_net\n",
    "        self.Vz = []\n",
    "        for p in self.V_net.parameters():\n",
    "            self.Vz.append(torch.zeros_like(p))\n",
    "        \n",
    "    def Pi(self, states=None):\n",
    "        if states is None:\n",
    "            S = torch.tensor(self.envs.state(), dtype=torch.float, device=device)\n",
    "        else:\n",
    "            S = torch.tensor(states, dtype=torch.float, device=device)\n",
    "        return torch.distributions.Categorical(logits=self.Pi_net(S))\n",
    "    \n",
    "    def V(self, states=None):\n",
    "        if states is None:\n",
    "            S = torch.from_numpy(self.envs.state()).to(device)\n",
    "        else:\n",
    "            S = torch.tensor(states, dtype=torch.float).to(device)\n",
    "        return self.V_net(S)\n",
    "    \n",
    "    \n",
    "    def ChooseAction(self, state=None, eps=0.):\n",
    "        '''\n",
    "         A = ag.ChooseAction(state, eps=0.)\n",
    "         \n",
    "         Choose an action, given the state.\n",
    "         With probability eps, it will choose an action randomly.\n",
    "         Otherwise, it will choose an action that yields the highest Q-value.\n",
    "        '''\n",
    "        #self.a_pointer += 1\n",
    "        #return global_A[self.a_pointer-1,:]\n",
    "    \n",
    "        if state is None:\n",
    "            state = self.envs.state()\n",
    "            \n",
    "        batchsize = len(state)\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            Pa = self.Pi(state)\n",
    "        A = Pa.sample().detach().cpu().numpy()\n",
    "        \n",
    "        # With prob eps, choose random action\n",
    "        #A_random = np.array(np.random.randint(0, high=self.n_actions, size=batchsize), dtype=np.float)\n",
    "        A_random = np.random.choice(self.n_actions, batchsize)\n",
    "        r = np.random.rand(batchsize)\n",
    "        ridx = r<eps\n",
    "        A[ridx] = A_random[ridx]\n",
    "        \n",
    "        return A\n",
    "  \n",
    "    def Episode(self, T=200, gamma=1., eps=0.):\n",
    "        '''\n",
    "         traj = ag.Episode(gamma=1.)\n",
    "         \n",
    "         Runs one episode, and returns the trajectory. The environment is\n",
    "         automatically reset at the beginning of the episode.\n",
    "                 \n",
    "         Inputs:\n",
    "           T      number of action steps\n",
    "           gamma  reward discount\n",
    "         \n",
    "         Outputs:\n",
    "           traj trajectory, as a list of lists of the form\n",
    "                [A, S, Snext, R, G, done, info]\n",
    "                A     action taken at state S\n",
    "                S     state\n",
    "                Snext state after taking action A\n",
    "                R     reward\n",
    "                G     future return (discounted)\n",
    "                done  flag to indicate episode is done\n",
    "                info  not sure ?\n",
    "        '''\n",
    "        #S = self.envs.reset()\n",
    "        S = self.envs.S\n",
    "        traj = []\n",
    "        bs = len(S)  # batchsize\n",
    "        for k in range(T):\n",
    "            action = self.ChooseAction(state=S, eps=eps)\n",
    "            Snext, r, done = self.envs.step(action)\n",
    "            traj.append(dict(action=action, S=S, Snext=Snext, r=r, G=np.zeros(bs), done=done))\n",
    "            S = Snext\n",
    "            if False: #done.all():\n",
    "                break\n",
    "                \n",
    "        self.envs.S = S\n",
    "        # Use that trajectory (and rewards) to update the policy parameters\n",
    "        # Process the return for all visited states\n",
    "        traj[-1]['G'] = traj[-1]['r']\n",
    "        for k in range(len(traj)-2, -1, -1):\n",
    "            traj[k]['G'] = traj[k]['r'] + (1.-traj[k]['done'])*gamma*traj[k+1]['G']\n",
    "        return traj\n",
    "    \n",
    "    def ComputeReturns(self, R, dones, gamma=1.):\n",
    "        G = np.zeros_like(R)\n",
    "        G[-1] = R[-1]\n",
    "        for k in range(len(R)-2, -1, -1):\n",
    "            G[k] = R[k] + (1.-dones[k])*G[k+1]\n",
    "        return G\n",
    "\n",
    "    def Simulate(self, trials=1, eps=0.):\n",
    "        time.sleep(1)\n",
    "        avg_k = 0.\n",
    "        for trial in range(trials):\n",
    "            observation = self.envs.envs[0].reset()\n",
    "            for k in range(self._max_episode_steps):\n",
    "                self.envs.envs[0].render()\n",
    "                action = self.ChooseAction(state=[observation], eps=eps)\n",
    "\n",
    "                observation, reward, done, info = self.envs.envs[0].step(action[0])\n",
    "\n",
    "                if done:\n",
    "                    print('Done after '+str(k))\n",
    "                    avg_k += k\n",
    "                    break\n",
    "        print('Average reward: '+str(avg_k/trials))\n",
    "        time.sleep(1)\n",
    "        self.envs.envs[0].close()\n",
    "        \n",
    "        \n",
    "    \n",
    "    def orig_REINFORCE_baseline(self, T=200, gamma=1.0, alpha=0.0001, alpha_v=0.0001, eps=0.1):\n",
    "        # Perform an episode and record the trajectory\n",
    "        traj = self.Episode(T=T, eps=eps(0))\n",
    "        bs = len(traj[0]['action'])\n",
    "                \n",
    "        # Update policy parameters\n",
    "        gamma_t = torch.tensor(gamma).to(device)\n",
    "        Pa_loss = torch.tensor(0.).to(device)\n",
    "        V_loss = torch.tensor(0.).to(device)\n",
    "        \n",
    "        Gmax = np.zeros(bs, dtype=float)\n",
    "        Gbest = traj[0]['G']\n",
    "        Ghist = []\n",
    "        \n",
    "        for k,blah in enumerate(traj):\n",
    "            A = torch.from_numpy(blah['action']).to(device)\n",
    "            S = blah['S']\n",
    "            G = torch.from_numpy(blah['G']).float().to(device)  # future return\n",
    "            G.requires_grad = False\n",
    "            dones = blah['done']\n",
    "            \n",
    "            Gmax = np.maximum(Gmax, blah['G'])\n",
    "            Ghist.append(blah['G'])\n",
    "            \n",
    "            # Policy network\n",
    "            Pa = self.Pi(S)  # returns a distribution object\n",
    "            \n",
    "            # Value network\n",
    "            V = self.V(S).squeeze()\n",
    "            \n",
    "            # Error\n",
    "            with torch.no_grad():\n",
    "                err = G - V\n",
    "            \n",
    "            # Recall that d(log(Pi)) = dPi / Pi\n",
    "            logPa_x_err = Pa.log_prob(A) * err  # * gamma_t\n",
    "            Pa_loss = Pa_loss + torch.mean(logPa_x_err)\n",
    "            V_loss = V_loss + torch.mean( (G-V)**2 ) * gamma_t\n",
    "            \n",
    "            gamma_t = gamma_t*gamma\n",
    "        \n",
    "        # Backproject gradient\n",
    "        self.Pi_net.zero_grad()\n",
    "        self.V_net.zero_grad()\n",
    "        Pa_loss.backward()\n",
    "        V_loss.backward()\n",
    "        with torch.no_grad():\n",
    "            #print(self.Pi_net[-1].weight.grad)\n",
    "            for p in self.Pi_net.parameters():\n",
    "                #print(torch.sum(p.grad**2))\n",
    "                p += alpha * p.grad\n",
    "            for p in self.V_net.parameters():\n",
    "                #print(torch.sum(p.grad**2))\n",
    "                p -= alpha_v * p.grad\n",
    "            \n",
    "        #return traj\n",
    "        #return torch.mean( torch.max(G, dim=0)[0] ).cpu().item()\n",
    "        return Gmax\n",
    "        #return np.array(Ghist)\n",
    "\n",
    "\n",
    "    def REINFORCE_baseline(self, T=200, gamma=1.0, alpha=0.0001, alpha_v=0.0001, eps=0.1):\n",
    "        \n",
    "        # Run and record a batch of rollouts\n",
    "        states = np.empty((T, self.envs.n_envs, self.state_dim))\n",
    "        actions = np.empty((T, self.envs.n_envs))\n",
    "        rewards = np.empty((T, self.envs.n_envs))\n",
    "        dones = np.empty((T, self.envs.n_envs))\n",
    "        #S = self.envs.reset()\n",
    "        S = self.envs.S\n",
    "        for t in range(T):\n",
    "            A = self.ChooseAction(state=S, eps=eps(t))\n",
    "            Snext, r, done = self.envs.step(A)\n",
    "            states[t] = S\n",
    "            actions[t] = A\n",
    "            rewards[t] = r\n",
    "            dones[t] = done\n",
    "            S = Snext\n",
    "            \n",
    "        self.envs.S = S\n",
    "        \n",
    "        A = torch.from_numpy(actions).to(device)\n",
    "        \n",
    "        # Calculate returns\n",
    "        G = self.ComputeReturns(rewards, dones, gamma=gamma)\n",
    "        G = torch.from_numpy(G).to(device)\n",
    "        G.requires_grad = False\n",
    "        \n",
    "        # Run the policy network and value network on the rollout\n",
    "        Pa = self.Pi(states)\n",
    "        V = self.V(states).squeeze()\n",
    "        \n",
    "        # Calculate error\n",
    "        with torch.no_grad():\n",
    "            delta = G - V\n",
    "\n",
    "        # Policy loss\n",
    "        logPa_x_err = torch.mean(Pa.log_prob(A) * delta, dim=1)\n",
    "        Pa_loss = torch.sum( logPa_x_err )\n",
    "        \n",
    "        # Critic loss\n",
    "        V_loss = torch.sum( torch.mean((G-V)**2, dim=1) )        \n",
    "        \n",
    "        # Backproject gradient\n",
    "        self.Pi_net.zero_grad()\n",
    "        self.V_net.zero_grad()\n",
    "        Pa_loss.backward()\n",
    "        V_loss.backward()\n",
    "        \n",
    "        # Gradient step\n",
    "        with torch.no_grad():\n",
    "            #print(self.Pi_net[-1].weight.grad)\n",
    "            for p in self.Pi_net.parameters():\n",
    "                #print(torch.sum(p.grad**2))\n",
    "                p += alpha * p.grad\n",
    "            for p in self.V_net.parameters():\n",
    "                #print(torch.sum(p.grad**2))\n",
    "                p -= alpha_v * p.grad\n",
    "\n",
    "        return torch.max(G, dim=0)[0].cpu().numpy()\n",
    "        #return G.cpu().numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cart-Pole"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "#env = gym.make('CartPole-v0')\n",
    "env = gym.make('LunarLander-v2')\n",
    "S = env.reset()\n",
    "T = 100\n",
    "envs = Vector_env(env, n=3)\n",
    "global_A = np.random.choice(2, (2*T,len(envs.envs)))\n",
    "ag = Agent(envs)\n",
    "ag.envs.reset()\n",
    "ag0 = deepcopy(ag)\n",
    "ag1 = deepcopy(ag)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "score = []\n",
    "counter = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "interval = 10\n",
    "iters = 400"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mysum = 0.\n",
    "for i_episode in range(iters):\n",
    "    Gmax = ag0.orig_REINFORCE_baseline(T=T, alpha=0.0001, alpha_v=0.0001, eps=(lambda x: 0.8))\n",
    "    counter += 1\n",
    "    mysum += np.mean(Gmax)\n",
    "    if counter%interval==0:\n",
    "        print(f'{counter}: {mysum/interval}')\n",
    "        score.append(mysum/interval)\n",
    "        mysum = 0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-1.4342, -1.3963, -1.3655, -1.3513],\n",
       "        [-1.4435, -1.3868, -1.3462, -1.3713],\n",
       "        [-1.4447, -1.3783, -1.3664, -1.3580]], grad_fn=<SubBackward0>)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ag1.Pi(ag1.envs.S).logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10: 27.34761611911149\n",
      "20: 3.0302463184835657\n",
      "30: 2.1947891321657154\n",
      "40: 0.3261674283950934\n",
      "50: 1.7415845909162346\n",
      "60: 5.161302393013194\n",
      "70: 3.8814019645798217\n",
      "80: 1.2995642039405257\n",
      "90: 7.124609706876109\n",
      "100: 1.5958808440999093\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "invalid multinomial distribution (encountering probability entry < 0)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-56-4a9a7f0be228>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mmysum\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi_episode\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miters\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0mGmax\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mag1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mREINFORCE_baseline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malpha\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.00001\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malpha_v\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.00001\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;36m0.9\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m     \u001b[0mcounter\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mmysum\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mGmax\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-38-60fd77bdfada>\u001b[0m in \u001b[0;36mREINFORCE_baseline\u001b[0;34m(self, T, gamma, alpha, alpha_v, eps)\u001b[0m\n\u001b[1;32m    240\u001b[0m         \u001b[0mS\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menvs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mS\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    241\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 242\u001b[0;31m             \u001b[0mA\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mChooseAction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mS\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0meps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0meps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    243\u001b[0m             \u001b[0mSnext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdone\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menvs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mA\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    244\u001b[0m             \u001b[0mstates\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mS\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-38-60fd77bdfada>\u001b[0m in \u001b[0;36mChooseAction\u001b[0;34m(self, state, eps)\u001b[0m\n\u001b[1;32m     88\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m             \u001b[0mPa\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPi\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 90\u001b[0;31m         \u001b[0mA\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPa\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msample\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcpu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumpy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m         \u001b[0;31m# With prob eps, choose random action\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/opt/anaconda3/lib/python3.8/site-packages/torch/distributions/categorical.py\u001b[0m in \u001b[0;36msample\u001b[0;34m(self, sample_shape)\u001b[0m\n\u001b[1;32m    105\u001b[0m             \u001b[0msample_shape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    106\u001b[0m         \u001b[0mprobs_2d\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprobs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_events\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 107\u001b[0;31m         \u001b[0msamples_2d\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmultinomial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprobs_2d\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_shape\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    108\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0msamples_2d\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extended_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msample_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    109\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: invalid multinomial distribution (encountering probability entry < 0)"
     ]
    }
   ],
   "source": [
    "mysum = 0.\n",
    "for i_episode in range(iters):\n",
    "    Gmax = ag1.REINFORCE_baseline(T=T, alpha=0.00001, alpha_v=0.00001, eps=(lambda x: 0.9))\n",
    "    counter += 1\n",
    "    mysum += np.mean(Gmax)\n",
    "    if counter%interval==0:\n",
    "        print(f'{counter}: {mysum/interval}')\n",
    "        score.append(mysum/interval)\n",
    "        mysum = 0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.2608, 0.2586, 0.2418, 0.2388],\n",
       "        [0.2620, 0.2577, 0.2415, 0.2388],\n",
       "        [0.2640, 0.2569, 0.2415, 0.2377]], grad_fn=<SoftmaxBackward>)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ag0.Pi(ag0.envs.S).probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[nan, nan, nan, nan],\n",
       "        [nan, nan, nan, nan],\n",
       "        [nan, nan, nan, nan]], grad_fn=<SoftmaxBackward>)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ag1.Pi(ag1.envs.S).probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD6CAYAAAC4RRw1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2dd3Rj13Xuv4NOAmAHOzkkp3AkTZWomdGMbDU7GskqllxiuUmyYiUueY7zEtvxey/LWXESJ7YTr5SXRLKaY1mxn4pVLVmyLdnSaHrXVLZhJ0CCJIjezvvj3guCIDougAti/9aaRQ4IEnvukB/33WfvbzPOOQiCIIjSQ1XsAAiCIIjsIAEnCIIoUUjACYIgShQScIIgiBKFBJwgCKJEIQEnCIIoUVIKOGOsgzH2G8bYWcbYe4yxr4iPf4sxNs4YOy7+uTX/4RIEQRASLFUfOGOsBUAL5/woY8wM4AiADwP4OAAn5/x76b5YQ0MD7+rqyiFcgiCI8uPIkSMznHNL7OOaVJ/IOZ8EMCm+v8gYOwugLZsgurq6cPjw4Ww+lSAIomxhjF2K93hGNXDGWBeA7QAOiA99mTF2kjH2KGOsNsHnPMgYO8wYO2yz2TJ5OYIgCCIJaQs4Y8wE4BkAf8I5dwD4dwBrAWyDkKF/P97ncc4f4pz3cc77LJYVdwAEQRBElqQl4IwxLQTxfpJz/iwAcM6nOechznkYwMMAduQvTIIgCCKWdLpQGIBHAJzlnP9j1OMtUU+7C8Bp+cMjCIIgEpHyEBPAHgCfAXCKMXZcfOybAO5hjG0DwAEMA/jDvERIEARBxCWdLpS3AbA4H3pF/nAIgiCIdKFJTIIgiBKFBJwgiJLglVOTmHX6ih2GoiABJwhC8Ti8AXzxyaP42eGxYoeiKEjACYJQPAvuAABByIklSMAJglA8Cx5BuF2+YJEjURYk4ARBKB5JwJ1eEvBoSMAJglA8DlHAFykDXwYJOEEQiocy8PiQgBMEoXgiAk4Z+DJIwAmCUDxS9wkJ+HJIwAmCUDxSBr5IJZRlkIATBKF4HB5BuKmNcDkk4ARBKB4pA/cEQgiGwkWORjmQgBMEoXgkAQcAly9UxEiUBQk4QRCKJ3qEftFH4/QSJOAEQSgehyeAmkotAOpEiYYEnCAIRcM5h8MTRGt1BQAa5omGBJwgCEXjDYThD4XRVisIOI3TL0ECThCEopEOMNtqBAGnVsIlSMAJglA00gGmJOBUQlmCBJwgCEUTycDFEgodYi5BAk4QhKKRtvG0VBsA0Dh9NCTgBEEoGqmEUlOpg1Gnpgw8ChJwgiAUjVRCqa7QwmTQUA08ChJwgiAUjWRkZTZoYNJrKAOPggScIAhFs+AJwKhTQ6tWwWTQUh94FCTgBEEomgVPANUVwhi9Wa+hPvAoSMAJglA0Dm8AVaKAm/RUA4+GBJwgCEWz4FkScCPVwJdBAk4QhKJxRJdQDBoseslOViKlgDPGOhhjv2GMnWWMvccY+4r4eB1j7HXG2EXxbW3+wyUIotxweAKoMkSVUHxBcM6LHJUySCcDDwL4n5zzywDsAvAlxtjlAL4B4Fec8/UAfiX+nSAIQlaiDzFNBg3CXFitRqQh4JzzSc75UfH9RQBnAbQBuBPAE+LTngDw4XwFSRBEeRIMheHyh1BVoQEgZOAAGVpJZFQDZ4x1AdgO4ACAJs75JCCIPIDGBJ/zIGPsMGPssM1myy1agiDKCoco1NE1cIAMrSTSFnDGmAnAMwD+hHPuSPfzOOcPcc77OOd9FoslmxgJgihTosfogagMnAQcQJoCzhjTQhDvJznnz4oPTzPGWsSPtwCw5idEgiDKFYco4NIhppFKKMtIpwuFAXgEwFnO+T9GfegFAPeK798L4Hn5wyMIopyJZOCVyzNwGqcX0KTxnD0APgPgFGPsuPjYNwF8B8DPGGMPABgB8LH8hEgQRLkiWcmuqIFTBg4gDQHnnL8NgCX48E3yhkMQBLHEQkwJhWrgy6FJTIIgFMuKQ0zqQlkGCThBEIrF4QlCq2YwaAWp0mvU0KlVJOAiJOAEQSgWaQpT6KUQoK08S5CAEwShWKKtZCWMetqLKUECThCEYok2spIw6bW0mV6EBJwgCMUSbWQlYdZr4PSRpSxAAk4QhIJxeFaWUEwGWuogQQJOEIRiETLw5eMqtFZtCRJwgiAUCeccDm9wRQmFMvAlSMAJglAkLn8IoTBfcYhppr2YEUjACYJQJLFTmBImvQbeQBiBULgYYSkKEnCCIBSJI4GAS5ayLsrCScAJglAmESOrODVwANQLDhJwgiAUSqISipkcCSOQgBMEoUhit/FIkCPhEiTgBEEokmSHmAAtdQBIwAmCUCgObxCMLW3hkaDN9EuQgBMEoUgcngBMeg1UquULwUx6ISMnAScBJwhCocQzsgIEO1mASigACThBEArFkUjAdbSZXoIEnCAIRbIQxwscAFQqRoZWIiTgBEEokkQlFEB0JCRPcBJwgiCUibBOTRP3Y+RIKEACThCEIkmdgYcKHJHyIAEnCEJx+IIheAPhhAJuNmjg9FIJhQScIAjF4fAI5ZFYIysJo45KKAAJOEEQCiTRGL2EyUBdKAAJOEEQCsThjW8lK2HSa6gPHCTgBEEokIUEToQSZrELhXNeyLAUR0oBZ4w9yhizMsZORz32LcbYOGPsuPjn1vyGSRBEOZFoG4+ESa8B54DbX96dKOlk4I8D2Bvn8X/inG8T/7wib1gEQZQzES/wJH3gABlapRRwzvlvAdgLEAtBEASANA4xaSsPgNxq4F9mjJ0USyy1iZ7EGHuQMXaYMXbYZrPl8HIEQZQLDm8QBq0Keo067sdpqYNAtgL+7wDWAtgGYBLA9xM9kXP+EOe8j3PeZ7FYsnw5giDKiQV3fCMrCcrABbIScM75NOc8xDkPA3gYwA55wyIIopxJNkYP0GZ6iawEnDHWEvXXuwCcTvRcgiCITHF4kwu4mbbyAEivjfApAO8C6GWMjTHGHgDwD4yxU4yxkwBuAPDVPMdJELIxPu/BHzxxGAtu8tJQKgueQMIhHiCqC6XM/VDi9+hEwTm/J87Dj+QhFoIoCL85Z8UbZ6fxu34bbtvSWuxwiDg4vAFsaDIn/HhkrRpl4ARRXvRbnQCAYyPzRY6ESIRwiJk4v9Rr1NBpVGVvKUsCTpQdF62LAIBjI3NFjoSIRzjMsegLJq2BA7SVByABJ8qQi9NCBn56wgFfsLwzOCWy6AuC88RGVhK0F5MEnCgzFtwBWBd92NpRA38wjLOTi8UOiYhhaYw+nQycBJwgyoZ+myDYH+9rB0BlFCWSaoxewmTQUB94sQMgiEIiHWC+b50FLdUGOshUIKmcCCXMlIGTgBPlxcVpJwxaFdpqK7C9swbHRikDVxqpvMAlaDM9CThRZly0OrHWYoJaxbC9oxajdg9si75ih0VEIW3jqa5MXQN3kYATRPnQb3VifaMJALC9swYAcHyUyihKYikDTz5nSDVwEnCijHD6ghif92C9OOG3qa0aGhWjg0yF4fAEoVaxiONgIkw6DXzBMPzBcIEiUx4k4ETZMCAeYK4TM3CDVo3LW6voIFNhLHiEKUzGWNLnSX4o5VxGIQEnyoaLooBLJRQA2N5RgxNj8wiFy3s5rpJIZWQlQZ7gJOBEGXHRugidWoXOusrIY9s7a+H2h3BhmgZ6lEIqK1kJM3mCk4AT5UP/tBPdDUZo1Evf9tJBJpVRlEOqZQ4SJvIEJwEnyod+mxPrmkzLHuusq0SdUUcHmQpCqIGnIeCRzfTla2hFAk6UBd5ACCN297L6NwAwxrC9owbHqJVQMTg8wQxr4OVrSEYCTpQFAzYnOAfWN65cErC9swb9Vmek/5goHpxzODwBVFWk3DVDm+lBAk6UCZIHyvqYEgogHGQCwAnKwouOLxiGPxROrwZOJRQS8Hzzm/NWvHXBVuwwyp6L006oVQxd9cYVH9vSXg3G6CBTCaTrRAgAlVo1GCvvDDz1fQqREz944yI457hug6XYoZQ1F62L6KqvhE6zMmcxG7TY0GgmYysFkK6RFQCoVAwmnQaL1IVC5Is5lx8T855ih1H2XLQ649a/JbZ31uDYyDw4p4GeYpKulayEyVDeW3lIwPPMnNuPGacf3kD5npQXG18whEuz7rj1b4ntnTVY8AQwOOMqYGRELJmUUADaykMCnkcCoXBkSoyy8OIxPONGKMwjHijxkA4yqQ5eXCQr2XTaCAHyBCcBzyPz7qXT8Yl5bxEjKW+kLfTJBHydxQSzXlP0gZ5gKIwXTkwgXKbeLAtuysAzgQQ8j8y7/ZH3x+fdRYykvOm3OsEYsNaSWMBVKoatHTVFz8DfumDD/3jqGA4N24saR7FY8AhibE7hBS5R7pvpScDzyFxUBj5OGXjRuGh1orOuEgatOunztnfW4NyUA25/8QRhXCy1Dc+WZy3e4Q3AqFNDq05PmigDJ/KG3bWUgVMNvHj0TztXjNDHY3tnDcIcODm2UICo4jPtEH7Rj9jL844tXSMrCepCIfKGVEJpqTZgfI4EvBgEQ2EMzjixLkkLocS2juIfZE4tCPs5R+zl+f2Srhe4hFmvgdMfLNszAxLwPCKVUK5orcbEQnn+QBabS3Y3AiGeVgZeZ9Shq76yqAeZ5Z6BOzIUcJNBA84Bd5m26ZKA55E5tx96jQprG42YnPeWbZZQTC5OJ/ZAicf2zlocGy3eQM+UJOBlWgPPuIQieoKX61q1lALOGHuUMWZljJ2OeqyOMfY6Y+yi+LY2v2GWJnMuP2ordWivqYA/FMaM01fskMqOfrGFMFkHSjTbO2tgW/RFDhMLzfSCF2oVw5w7EOmJLicWvcG0xugljHp15PPKkXQy8McB7I157BsAfsU5Xw/gV+LfiRjm3AHUGnVorakAgKKJQjlz0epEW00FjCk2nEtsL2Id3OULYtEXxKbWKgDAaBmWUTLNwM2G8t6LmVLAOee/BRDblHongCfE958A8GGZ41oVzLv9qK3UkoAXkYvTzrTLJwCwscUMvUZVFAGX6t9Xd9UBKD8BD4bCcPqCaXmBS0TWqlEGnhFNnPNJABDfNiZ6ImPsQcbYYcbYYZutvGxV7W6hhNJWKwg4tRIWllCYY8CWXguhhFatwpb26qI4E0r17z5RwC/NlpeAS2WQzGrg5e0JnvdDTM75Q5zzPs55n8VSXpaq8+4Aaiq1qDJoYdZrqJWwwIzPeeALhpO6EMZje2ct3ht3wB8M5ymy+EgZ+PomE2oqtWXXiZKpkRVAm+mzFfBpxlgLAIhvrfKFtDoIhznm3X7UGXUAgLbaCprGLDCSB8raDDJwALi8pQr+UBhDBXYmlHrAm6sMWFNXWXYCHjGyyuAQcykDJwHPhBcA3Cu+fy+A5+UJZ/Ww6A0izIGaSkHAW2sqVlUNPBTm+MmBkYj5kBK5KK5RS2ZiFY/eZiFjPz+9KHtMyZh2eGHWa2DUa9BRV1l2NfBIBl6ZSRdKee/FTKeN8CkA7wLoZYyNMcYeAPAdAB9kjF0E8EHx70QUdnEKs1b8ZmyrqVhVNfDXz0zjm8+dwkunJoodSkIuTjvRVKXP6JYcAHosRqhVDOenHHmKLD5TC140VRsAAJ11lRib8yAYKmwZp5hkU0LRaVTQaVRwFtG/ppikPO7lnN+T4EM3yRzLqmIuIuBLGfiCJwCnLxi57StlHntnCAAw7VBub3u/dTHj+jcA6DVq9DQYcX6qsBn4lMOL5qolAQ+GOSYXvOioqyxoHMXCIToRZlJCAcRxesrACTmRfFBqjZKACz+YqyELPzPhwIEhobPUtqjMuj7nHBetzozLJxK9zWacK7CAWx1eNEkCXi+IdjmVUbLJwIHyXuqw6gWcc447//Vt/OTASEFfd84lfDNKJZT22tXTC/74viEYtCq011bAqtAMfGLBC7c/lFEPeDQbm80Ym/MUTBjCYQ7rog9NVXoAQgYOlJcnyoInAK2awaDNTJbK2RN81Qu4zenDibEFPLFvuKCvK5VQog8xAZR8K6Hd5cfPj0/g7ivbsdZignVRmQJ+UTyAzKaEAgC9zcI05IUCHWTOuHwIhjmaxRp4S3UFNCqGS2Uk4A6vMIXJGMvo80z68t1Mv+oFfHhG+AE4P71Y0JrmnNsPtYqhSuxTbTQboFGxki+hPHVwBP5gGPfv7kKjWQ+rQkso/WIHSiZDPNH0NomdKAX6npkWWwilEopaxdBeW1F2GXgmToQS5jL2BF/1Aj4044y8/+KJwnVMzLkDqK1cyibUKobmakNJC3ggFMZ/vXsJ165rwPomMxqr9Jhx+hFSoMvixWknGky6yBlEprTXVqBSpy6YgEtTmNIhJgB01htlr4H7giHFumI6PIGMDzCB8t7KUwYC7oZWzXBNTz1ePDlRMJvQebc/Uj6RaCvxXvDX3pvClMOL+/d0ARDuKkJhvmzzkFLot2V/gAkIOzI3NJlxrkCthBEBr44S8Dp5M/CRWTdu/N5b+PbLZ2X7mnLiyNDISsKo15Cd7GplaEbYh3jX9jZcmnUXbF2W3eWPHGBKCL3gyiw5pMNj7wxjTX0lbugVrG8azcKBm9LKKJxzXJzOroUwmo3NZpyfWizIL32rwwsVAxpM+shjnXWVmHcHIt0ZuTA+78Enf7gf4/MeHLmkzIXJdrcfNRkM8UiYDFQDX7UMz7jR3WDEzZuaoVWzgpVRBB+U5Rl4a00FphzekhzOODk2jyOX5nDvNV1QqYSyUKPYMaG0ThTrog8ObzCnDBwQWgnn3AHYCnBQO7XghcWsh1q1dIAndaLkWkaZdnjxqYf3Y8ETwPvWN+DCtFNxZZRgKIzJeW+kWysTzHoN/MEwfMHy28qzqgU8HOYYnnWhu8GI6gotrtvQiJdOThbkm3fOHScDr61AKMwxrdDOjWQ8/s4wjDo1PtrXHnms0Szc7istAz87KZQ9NjbnloEXcqQ+eohHorPOCCC3VsIZpw+f+uEB2BZ9eOJzO3DblhZ4AiGMzinrcHRywYtgmEd+aWWCNBjn8pGAryomHV74gmF0NQg/CHdsa8WUw4tDw/m9heScR5Y5RFOqrYTWRS9ePDmBj/V1LDtkspiVmYGfnRQEd6PYCpgthexEmY4a4pHoqBO+X7IV8Hm3H5/+4QGMzbnx6H1X48rOWmwocHdNukj/RumXViaYDOXrCb6qBXzIJrjJdYsC/oHLGlGhVePFk/kto7j9IfiD4cgYvURbTWn6gv/kwAgCIY7PXrNm2eMGrRrVFVrF9YKfm3KgraYiI1OkeNSb9Ggw6QsykTm14F12gAkAZoMWdUZdVr7gDm8An3nkIAZnXPjhZ6/Gzp56AMB6UcAL1d+eLhEBr88+A19M4gl+YnQeX/3p8YJbBOeb1S3gs8sFvFKnwU2XNeKVU1N5rUPPxRhZSUjj9KXUieIPhvHj/SO4odeCnjh7JZXYC3520pFz+URCOsjMJx5/CA5vcEUGDgh18Exr4E5fEPc9ehDnphz4j09fiWvXN0Q+ZtJr0FFXUXCbgFSM2IVusdgyUjpE1qolycB/8MYFPHdsHG+eX13O16tbwG0uVGjVaDIvfVPcsbUVdpcf7wzM5u1150WL1dhDzEqdBnVGXUkJ+MunJjDj9OG+Pd1xP95YpVdUBu4LhjBgc+GyltzKJxK9zWZcmF7Ma697vB5wic4MfcE9/hA+9/ghnBhbwL/ccyVu3Ni04jm9TWZFZuDttZXLDnHTJVIDT+BIODbnxpsXhG1gzxwdyz5IBbKqBXx41oU19ZWRrgkAuK7XArNBk9duFCkDr4szRNJaUzrDPJxzPPbOMHosRrxvXUPc5zSaDYqqgV+cdiIU5tjYIk8G3ttshi8YzutEpLSJJ1EGPj7vQSDNO8a/fvkMDg3b8Y8f34q9m5rjPmdDkxmDNpeiygmjdnfWrouSJ3iirTw/PTQKBuDWzc349Tkr5hQ4t5Atq1rAh2Zc6LEsPxTRa9S4+YpmvHZ6Ct5Afk6tpcGW2BIKALRWV5TMIeax0XmcHFvA/bu7lv0SjKbRrIdt0VewAalUSKUB2TLwyKFf/gZ6piNDPPoVH+usq0QozDGZxvwA5xyvn5nGbVtacee2toTP6202IxjmGIyaUi42I3Y3OusybyEEkm+mD4TC+OmhUVzf24gv37AegRDHCwWcyM43q1bAA6EwRu1udNWvPNW+Y2srFn1BvHUhP0uWE5VQAKGVcGLeoxjBS8Zj7wzDbNDg7ivbEz7HYtbDHwrLMmwiB2cnHTBoVXH/37NhQ5MZjCGvNeOphSQZeH36roT9Vidsiz5cu64+6fMi7ZEKqYMveAKYdweyaiEEotaqxcnAf3V2GtZFHz65oxOXt1bhspYqWcsoLl8Qb5634nuvnceTBy5h0OYs6M926W8WSMDYnAfBMI8cYEaze2096ow6vHhiAjdfEf82MxciToRxxoLbairg8ofg8ARz7pLIJyOzbrxyahL37+6K3KLGo7FK6gX3xf2FVWjOTTnQ22TOqpYajwqdGmvqKvMqdlMOL4w6NcxxfEAysZV9p38GALB7bfxyl0RPgwkaFVNMHXw0hxZCAKjUqcFY/Az8yQMjaKk24PpeYaH6R65sw7dfPitM6jZlXmbzB8M4PjqPd/pnsG9gBsdH5xEIcTAGSLrdVKXHNT312L22Adesrc/rQo5VK+CSiVVsCQUANGoVbt3cjKePjMHlCyYVqGyYdwdQZdBAo155gyO1Eo7Nu1FdWS3r68rJ/32zH2oVw+ff35P0eY1RveAbsviBkBPOOc5OLuKDl608uMuF3jx3okw7llapxdJUZYBOrcIle+oFy+8MzKKjriKlYOg0KvRYCr9xKBFLPeDZCR1jTLCUjcnAR2bd+N3FGXz1AxsiP4t3bmvD3/3iHJ4+Ooa/uOWytF/jl+9N4ckDIzg0bIfbHwJjwOa2ajxwbQ/2rKtH35o6TDm82Dcwg3cHZvF2/wx+flwo1bTVVOCatfX4g/d15zybEMsqFnDhmyLRrfQdW9vw4/0jeOPsdNJ6YTbYXf6ELnitkV5wL65oVaaAj8258fSRMXxqZ2fc2/poJAGX6rjFxLbog93lx2UyHWBK9DZX4fUz0/AGQjBo1bJ+bUDsAU9wnSVb2VSthMFQGPsHZ/GhzS1pveaGJjNOjM1nHGs+kAS8I8saOCCuVYvJwJ86NAK1iuH3r+6IPGYx63H9Bgt+fmwcX7t5Y1p3aiOzbnz5J8fQWKXHR69qFzLrnvoVd9DdDUZ0NxjxqZ1rwDlHv9WJfQOzeHdgFm+cncY9Ozqz/vclYhULuBNVBk3cThAA6FtTi5ZqA148MSm7gM/FcSKUaJM28yhslDmaf39zACrG8EfXr0353OgSSrE5I43Qy3SAKbGx2YwwF2rMm9rk/6U77fBhZ3ddwo931qduJTw94cCiN4jdCbqFYultMuOlk5N5uQPNlBG7G3VGXdwSUrqYYjzB/cEw/t/hUdy4sXHFgNRHrmrHr85Z8Xb/DK7bYEn5tb/z6lmoVQzPfGF3yoRGgjGG9U1mrG8y497dXXmz71i1h5iSiVWi7R4qFcNtW1rw1gUrFtzyHsDNi17g8ag36qDTqDCxUPyMNR4T8x787PAoPtbXjpbq1BmRSa+BUadWxDCPNEJ/mcy3qVJpKB8HmcIqNW/kF2E8OusqMZJiGnPfgFD/vqYn+QGmhHSQqYQ6eC4thBJGvWZZH/jrZ6Yx4/TjkztXZr03XdaI6gotnjmS+jDz4JAdr5yawh9dtzZt8Y6HSsUSdnLlwqoV8KEZV9wDzGhu39qKQIjjtfemZH3tObcfdQkycMZYQX3BMz0R/8+3BsA58IU0sm+JxiqDIjLwc1MOtFYbZD8c7qqvhE6jyksrod3tRyDE0Vy1soVQorOuEg5vMLIoOx77+mfR22SO+NOkQkkCLrQQ5ibgsTXwnxy8hLaaCrx//coMW69R4/atLXjtvSk4vImTt3CY469fOoPmKgM+//74g2zFZlUKuDcQwsSCJ2JilYjNbdVYU18pe1/onCtxCQUQFzsUoBf80beHcMP33sTkQnqvNe3w4qlDo/joVe1or03/B8pi1sOmgGGes5MO2fq/o9GoVVjfaMpLBi61EMbe5kfTkaITxRsI4dCwHbtTtA8u+5q1lajQqnF+qri94MFQGONznqx7wCXMUZvph2ZceKd/Fvfs6EhY4/7Ile3wBcN45eRkwq/58+PjODW+gK/t7UWlTpnV5lUp4Jdm3eAcKTNwxhju2NqKfQMzspUA/MEwXP5QwhIKUJhpzHCY45G3hzA868bnf3QYHn/qoaX/fGsQoTDHF69fl9FrKcEPRRqhl2sCMxZppF5ukk1hSqxJ0Qt+dGQOvmAYe1K0D0YjbBwy4fx0YTYOJUKykV2TZQuhRPRm+qcOjkCjYvh4X0fC52/rqEGPxZiwJ9ztD+IfXj2PLe3V+LDMZ2RysioFfGhmuYlVMqQDzD/+yTFZ1jJJt7k1SXYxttVUwrroy6sB/cFhO8bnPfjIle14b8KBP3/6RNJyim3RhycPXMJd29sydoRrNBe/hCKN0OcjAweEg8xphy9pGSMb4q1Si6WjNrmA7+ufhVrFsLMn8UFoPDY0mYuegS91oORaQtHC6QvCFwzh6SNj+MBlTUnPFRhj+MiV7Tg0PIdLsytbNB/67SCmHF787w9dnpfatVysagFPVUIBgHWNJvzT72/D4UtzuPfRg1hMUhNLhznxQDRRDRxYciWcyuNB5nNHx2HUqfHtD2/C1/duxEsnJ/Gvv+5P+Pwf/m4QgVAYX7ohs+wbEAyt3P5QURfLyj1CH0uveDAqdxllesELxgCLKXHt2qjXoMGkS3iQuW9gBlvaqzPu4uhtNmPG6cOss3i/fHOxkY3GJJZQfnFqCnaXH5/albpl7+4r28AY8OzR8WWPTy148Z9vDeLWzc3YkaQ7SAmsSgEfnnGhwaRLe8P1ndva8M+f2I7jo/P4zCMHcxoLT+aDIrHUSpifMoo3EMIrpyaxd1MLKnRq/OH7e3D39jZ8//ULePX0yprfrNOHH717CXdua0vrriWWpWGe4pVR5B6hjyVfyx2mHT40mPRxh76i6UjgSrjoDbl4bpIAABy9SURBVODE2AJ2r02//i1RyI1DicjFRjYas9gK+fDvBtFZV5lWOamlugJ71jbg2WNjy9r8vvvaeYTCHN/Ym/6gT7FYlQKeTgdKLB/a0oJ/+9SVeG9iAZ955EDWt8qREkqKQ0wgf77gr5+ZxqIviLuvFMpDjDH87d2bsa2jBl/96QmcmVhe93zk7SF4g6Gssm8gerWafJnc5IInIx9suUfoY2mq0qO6Qit7Bh5vlVo81iQQ8INDdoTCPKP6t4T0S+lCEScyc7GRjUbqZX9vwoF7dnSmXfb4yFVtGLV7Ilu6To0t4JmjY7h/T1fOdwWFYHUK+Kwrq0zs5iua8R+fvgrnJhfxyYcPRLLpTJBKKLXGxBm4VO/M14b6546No6XagF1RPcEGrRoPfeYqVFdo8fkfHcaMeNs87/bjiX3DuG1La9ZLgCPLjWUU8K8/cwqfffRgWm2Q0gi93GPK0TDGxJF6eQ/94q1Si0dnXSUm4tjKvtM/C71GhSvX1Gb82hazHjWVWpyfLl4dXI4ecEAooQCAVs3wsb7E5mux3HxFM4w6NZ45OgbOOf765TOoN+rwpRuzS2YKTU4CzhgbZoydYowdZ4wdliuoXFj0ClvEu+N4oKTDTZc14eF7+zBgc+Keh/ZHhC5dlrbxJM7A9Ro1Gs16jM/LP4054/ThrQs23LmtbUVW01hlwMOf7cOsy4cv/PgI/MEwHn17CC5/CH+cwzdsPkoo5yYdGJpx4cTYQsrn5muEPpaNzWZcmJbXbW7K4Y1rIxtLR10lwnxl2W3fwAz6umqzGvFnjKG3Sf5fSplwaTZ7G9lopBLK713RjIYk5wmxVOo0uHVzC145NYWfHx/HwSE7vvrBDWmXX4uNHBn4DZzzbZzzPhm+Vs5I+wO7c6iFXrfBgkfvuxqX7C584qH9GQnTnMuPCq065Q9Ua01FXjLwF45PIBTmkfJJLJvbq/Hdj27FoeE5/PnTJ/DYO8O4dXNzTkZU1RVa6DQq2TJwhzcQ+Vo/Pzae4tn5G6GPpbfZDKcvKFvpyxsIYd4dSKuEEs+VcMbpw7mpxZTug8nozcMvpXRZcAew4Ank3EIICK2WOo0K9+/uyvhzP3JVO5y+IL7+9ClsaDLhE1cnbj9UGiVRQjk35cDzx1P/IAPAoNRCmGUGLrFnXQMev38HJuY9+MRD+9Ne/jCXZIw+mrba/ExjPndsHFe0ViUV5Nu3tuKPb1yH549PYNEXxJdvWJ/TazLGhF5wmTLwAatwS19docVLJydT7i/N1wh9LHIfZKbTAy6xRkxIogX8XXEtYDYHmBIbmoRfSsWwdhidk6eFEAB6LCac+aub0deVedfIjq46tNdWwB8K43996PKUB8pKItdIOYBfMsaOMMYejPcExtiDjLHDjLHDNlt2CxR+vP8SvvHMKbgT7LyLZlgUcDl+q+/qqcff3rUZgzOuSJaXivkkRlbRSOP0cmY+/dZFnBpfSLqAQeKrH9iAT+7sxH27u3B5a+7CJwzzyJOBD9iE/8MH39+DGacP7w4m31+arxH6WDY0y+uJMi1OrybrAZdoNOuh06iWHezuG5iBWa/B5hwMtjY253/jUCJytZGNJVvhVakYvrZ3I/7wup60zK2URK4CvodzfiWAWwB8iTH2/tgncM4f4pz3cc77LJbsLs7tW1rhCYTw+pnplM8dmnGhtdqACp08tp+b24UfjiFbaj9mQPRBSTLEI9FWUwF/MIxZGffzPXt0HGqVMF2aCpWK4W/v2oxv3XGFLK8t5zDPgM0JjYrhvt1dMBs0eP54cquDfI3Qx1Jl0KKtpkK2DHwqgwxcpWLoqK2IlAgB4QBzZ099Thnj+shdRe4Hmc8eHUPft99IK9EC5LGRlYs7trZm5A+uFHIScM75hPjWCuA5ADvkCCqWq7vq0FwlWL+mYmjGldYAT7p01gktTtJwUCrm3AHUpJEJSr7gcvWCh8McPz82jvetb0jb0EhOGqvkLaF0NRhh1Gtwy6ZmvJpkf2m+R+hjkXOkfjrJKrV4RG+oH7W7MWJ3Y08G/ifxqK7QoqXaIMu/6dXTU5hx+nDk0lxaz5fDRrbcyVrAGWNGxphZeh/A7wE4LVdg0WRi/ZpND3gytGoVOusq014AO+f2J+1AkZCmMeXyRNk/NIuJBS/u2l4c34ZGsx4Ob1CWRdEDNifWimcYd25rg9MXxK/PWeM+N98j9LH0NpsxYHOmvSU+GVMOLyq0alQZ0jNKWlNvxKjdDc55pP69J03/72T0NptzLguFwxwHxV7q/SlKXhJytRCWM7lk4E0A3maMnQBwEMDLnPNX5QlrJRHr1zOJrV/nXH4seAKyCjgA9DQYMZhGCSUU5ljwpHeI2V4jfOPKdZD53NFxmPQa/N7l8u/4TAdpmMeWYxklEArj0qwbay1CT/qunnpYzPqEh9iS8OSzBzyajc1mBEI8re+HVAgthIaEnvWxdNRVYtEXxLw7gHcGZtBg0mN9lr370fQ2mTFgdaY8LE7GBesi5t0BMAbsH7Sn9TlCCyEJeC5kLeCc80HO+VbxzxWc87+RM7BYtrRXo7OuEi8msX4dzMDEKhO6G4wYnnWl3Krh8ATAORKuU4umqkJYhCCHgHv8Ifzi9BRu2dQsW+0/UyyRYZ7cyiiXZt0IhnlEwNUqhtu3tOI352xxLQ7OTjqg16hk/z9PxNJyh9wP/aYXvGhK4gMeiyR2w7Mu7BuYxe619WmLfzI2NJnhD4UxnGJpRDIOiKL9oc0tODE6n7IOHgyFMT7vwRoS8JwomX4Zxhhu39qCd/pnEg7XDGdgYpUJPRYTvIEwJlPUeO1pDPFIMMaEVkIZauC/PDMFpy+IuxL0fheC6OXGuTBgE0pVa6Myyzu3tcIfCuO10yvvvs5NOdDbnL8R+ljWWoSN7nIcZKY7Ri8hCfivz1lhW/TlXP+WiHii5PBv2j84i7aaCnz0qnYEwzxlHXxywYtQmFMGniMlI+CAsIg4zIFfnIp/mDk044JaxSL2m3IhZXeDtuR18CUflPQOZVprKjCR5rKFZDx3bByt1Qbs6pbnBzob5PJDiQh4VB//lvZqdDcY8fyJ5WUUaYQ+3/3f0ci10Z1zDqvDl9GaLknsfnZ4FAByGuCJZl2jCSqWvakV5xwHh+zY2VOHvq46qFUskpEnQi4b2XKnpAS8t9mMDU2mhN0oQzMudNRWQKeR958liUmqTpQ5l+iDkkYGDgithMmmMUNhjkPD9qSeLLZFH353cQYf3t5WVN/ieqMOahXLeTv9gNWFpir9ss6EpcUbs8u+vjRCX6gOFImr1tTi3cHZnHapzrkD8IfCGQl4hU4Ni1mPaYcPnXWVsomfQatGV70xa1OrfqsTsy4/dnXXwyT2pac6yJTLRrbcKSkBB4Se8IPD9rjdG3K3EEpYzHoYdeqUB1eSD0o6feCAkIHbXf649cJ9AzO4/V/exsf+413s+Js3cN9jB/Hs0bEVfuUvnEg+Ol8oVCoGiyn3YZ5+mzNS/47mjm2t4BzLzkCk4apCdaBIfGZXF9z+EJ46NJL110hnlVo8pCxcrvKJRG+zOesMfP+QkG1LCyV29dTjxFjyOrhcNrLlTukJuDik8nLMLjvOOYZn5W0hlGCMocdiihySJmIuwxKKZCsbnYUP2pz4/I8O45MPH8CCJ4C/u3sz/uB9Pbg47cSf/uwE+r79Br7w4yN45dQkvIEQnj06hs1t1VjXWNgsNB6NVbkJOOccg9b4Ar7WYsLmtupl+0sLNUIfy+WtVdi9th5P7BvOup0wkzH6aCQBv0am8onEhiYzhmddWbWBHhicRXOVIRLbrp46BEIcRy/NJ/wcuWxkyx1lbupMQleDEVvaq/HiyQl8/v09kcetiz64/aG8dSN0NxhxdCT5wcycOwCNisGkT++yRhY7zHvQYNLhn3/Vjx+9OwyDVo2v7e3F5/Z0R0yxvr63F0dH5vDiiUm8dHISvzg9BaNODZc/hL+87fKc/m1y0WjWYyyHQ1nbog+LvmBCW9s7t7Xi2y+fxaDNiR6LqWAj9PH4g/d143OPH8YrpyYja/kyIZ1VavFYazFCxXLzP4lHb7MZnAvlkE0ZjOZzzrF/0I4965Y6YqQ6+P7BWVy7Pv4vmpFZ6gGXg5LLwAGhjHJybCHSdQIgUt7Il4D3WIwYn/ckzVAkH5R0W7ukaczH3hnCdd99E4/vG8LH+jrwmz+7Hl+8ft0yR0PGGK5aU4dv3XEF9v/FjfjxAztx25ZWbO+swYeLNLwTi8VsyKkPvD9ygBlfwG/f2grGEMnCz0468u5AmIjrNzSix2LEI28PZeVnMyWuUmvMcGr23t1deOYLuzOyTE2H3ix9XgZnXJhx+pZ5z5v0GmxKUQcfsbuphVAGSlLAP7SlBQDw0sml2+lhcTFpvlZqdTcYwTmWeVHEMucKoC7JIodYmsx6aFQMb563YUt7NV75yvvwd3dvTjkKr1GrcO36Bvz9R7fguS/uSbvmnm8azXrMuvxZlxUkE6u1jfH/D5uqDLimpx7PH5+ANyCM0OfbAzwRKhXD/Xu6cXJsAYfTHB2PxrroRb1RD22GPiZmgxbbOzNf3pCKNXWCHWumI/VSt8nOmN2Ru3rqcGJsHh7/yoRHspGlFsLcKUkBb62pwNVdtcvqoUMzLug0qkhWKzdSVjiUZKTenqYToYRGrcL3P74Vj91/NX70uR0FmybMF9JmnkyXYEgMWJ2o1KmTHmzdua0VQzMuPHdsHKEwL+o1+8iVbaip1OKR3w1l/LlTGQ7x5BuNWoV1FlPG7ZEHhmZhMetX3Pnu6qkX6uBxyo5y2siWOyUp4IDgHnZh2hn5hhuacWFNXf4ORaTuloEknSjzbn9aY/TR3LmtDTf0NsoyUVdsIr3gWQ7zDIgdKMmuxd4rWqBTq/CDNy4AKHwHSjSVOg0+uaMTvzwzlXBjfCKmHD7FdWBsbDZnJOCccxwYtGNnd92K/7O+NbWROngsctvIljMlK+C3bG6Bii21lcltYhWLSa9BU5U+aS+4sMxBGeWMYhCZxsyyDj5gdabcy1ldqcX1vRZMO3zQa1ToKnIf8b27u6BWMTy2L7MsfNrhRVOGB5j5ZkOzGVMOb9r97SN2N6YcXuzsWXmgajZoE9bBlWQjW+qUrIA3mPTYs64BL54U+qBHZt1598PobjAmnMbknAsZuELq0cWgMQc/FJe4FWZtGpuUpEPb3mZz0benNFUZcNuWVvzs0Cgc3vSEzxcMwe7yKy4Dj2ypt6aXhUvivKs7/hacXd11OD66sg5ONrLyUbICDgjdKJdm3Xj19BT8oXDeBbzHYkqYgTt9QQRCPOMSymqiwaQHY9mVUKTrmqgDJZobNzaiyqDBlvbsN9HIyQPXdsPlD+GnB0fTer50fRQn4GInyuHh9A5lDwzaUW/UJbxrkurgx2Lq4CPkQigbJS3gN1/RDK2a4V9+fRGA/CZWsfQ0GDHnDmAuzmj7vHjbmckh5mpDq1ahrlKXVQbeb11pYpUIg1aNF//4Wvz5zRszfp18sKmtGju76/D4vuG0LFkjm3gUVkJpranANT31+OHvBuH0pd6qc0D0P0l0ZtHXVQsVW+kPPmInAZeLkhbw6kotrtvQGOld7cl7Bi6aWsXJwucycCJczTRWGbLKwAdsTqiYsF08HdbUG1FdoZy7nQeu7cb4vAevvpfYr15iaQpTOV0oEl+/ZSNmXX788HeDSZ83andjfN6DnUkM1MwGreiLsmRsJdnIkoDLQ0kLOADcvlXoCTeKRj/5pLtByA7j1cHnxAw8kz7w1Ui2y40HbE6sqTdCrymOn3mu3HRZE9bUV+KRt1MfZkZ8UBRWQgGAbR01uGVTMx7+7SBmk7SDHojxP0nErp76ZXVwspGVl5IX8A9c1gSDVoWuBmPeW/E6aiugSbAfUyqrlHMJBZAEPPMSyoDVldYBplJRqxg+t6cbx0bmU3phTzu80GtUirqDiObPbu6FNxjGv/6mP+Fz9g/OoqZSiw0pPHh29dTDHwpH6uBkIysvJS/gRr0Gf37zRtx7TVfeX0ujVqGzvjKuKyGVUAQaq/SYcfoRSrG9KJpQmGNoxpXWAaaS+ehV7agyaPBoiix8yuHLaJVaoVlrMeFjV7Xjyf0jGLXH728/MDSLHV11KS2MY+vgZCMrLyUv4IBQf/z41R0Fea2ehvidKHPiPkClZlWFotFsQCjMk3qYxzJqd8MfCpe8gBv1GtyzsxO/OD2ZUPgAaZWa8son0fzJBzaAMeCfXr+w4mMT8x6M2j3L/E8SEekHF0suZCMrL6tCwAtJj8WIoVnXigxz3u1HdYW27O0xl4Z50i+jxFujVqrce00XGGP49CMH8JfPn8bLJydXGHxlukqtGDRXG3Dfni48d3wcZyeX7/88MCRk06nq3xK7eupxfGQe3kBIcCEkG1nZIAHPkJ4GI/zB8IqFEnaXv+zLJ0D0ME/6B5nx1qiVKq01FfjB729DZ10lnj4yhi/95Ciu/ps3cNP338Q3nzuF54+PC1OYCuxAieWL162DWa/Bd187v+zxA4N2VBk0afvQ7Oqpgz8UxtGROYzYyUZWTkrOD7zYRPZjzriWfSPOuwNpL3JYzUh+KLYMWgkHrC40mHSr5gD49q2tuH1rKwKhME6PL+DAkB0HBmfx4vEJ/OSAsMUnX6ZrclJdqcUXrl+Hv3/1HA4O2bFDnLg8IL6fbhbd11Un1sHtGLG7sa2jJp9hlxUk4BnSI7kS2py4boMl8vic26/4umYhsGRZQukp8fp3PLRqFbZ31mJ7Zy3+6Lq1CIU5zk468N7EAvZe0VLs8NLivt1deHzfEL7zi7N45gu7YV30YWjGhU/u6Ez7a1SJdfBfvjdFNrIyQyWUDGkw6WDWa1YM88yXuZGVhEGrRpVBk3YJhXOecA/makOtYtjUVo3fv7qzKFuEsqFCp8ZXbtqAoyPzeP3MdKSbJN36t8SunvrIwB2VUOSDBDxDhP2YxhWdKEINvDR+KPNNY5Uh7e30dpcf8+5AShdConh8vK8dPQ1GfPe183h3YBZmvQaXZ2jjG73wgTJw+SABzwLBlXBJwL2BEDyBUFk7EUbTlMFy48gWnlVwgLla0ahV+LObe3HR6sTTR8bQ11WbsQukVAcHqAdcTkjAs6DHYlq2H3PJyIoycEA4yEzXD2UgxR5MQhncsqkZW9urEQzzuP7fqaiu0OKK1mrUG3VpL/0mUkMCngVSJ4pURpGmMOuoBg5A6AW3LfrSWvY7YHVCr1GhrQS6MsoZxhi+eetlMOrUuHFjY1Zf40s3rMMXb1gnc2TlDf0qzALJlXBoxoXLWqrIByUGi1kPfyiMBU8g5TXpFztQUo1kE8VnZ089Tv/VzVlbAOzd1CxzRARl4FkQ6QUXb/8lJ8LaMncilGgU2ynTqYMP2FKvUSOUg1L9W8qVnAScMbaXMXaeMdbPGPuGXEEpnUqdBi3VhkgrIRlZLScyTp+iDu4NhDA256EDTILIkqwFnDGmBvBvAG4BcDmAexhjl8sVmNKJ7kSZd0slFMrAgfT9UIZmXOCcDjAJIltyycB3AOjnnA9yzv0A/hvAnfKEpXx6LMKCY8457K4AjDp1yS4jkJt0SyjUgUIQuZGLgLcBiN7iOiY+tgzG2IOMscOMscM2my2Hl1MW3Q0mOLxBcRDFTweYUZj0GlTq1ClLKP1WJxhbOhQmCCIzchHweKcZK/rGOOcPcc77OOd9FoslzqeUJtGdKHNuPx1gxpDOZp4BmwvttRUwaOnOhSCyIRcBHwMQvUWhHcBEbuGUDj2RThQX5sgHZQWNZkPqEoq1PDxQCCJf5CLghwCsZ4x1M8Z0AD4B4AV5wlI+7bWV0KoZBqUMnAR8GY1VeliT+KGEwxyDMyTgBJELWQ/ycM6DjLEvA3gNgBrAo5zz92SLTOGoVQxr6oWDzDkyslqBkIFbE358YsEDb6D016gRRDHJaRKTc/4KgFdkiqXk6Gkwot/qhMMbpEPMGBqr9HD7Q3D6gnG9L/qtQgcKDfEQRPbQKH0OdFuM+OWZaQBAHTkRLkNaGXbT999EpU4DvUYFvVYNg/hW2hNJQzwEkT0k4DkgHWQCNMQTy/UbGvHAtd1weALwBsPwBULwBsPwBkJY8ATAOcfeK5rpFx9B5AAJeA5ErwGjQ8zl1Bp1+D+3lc1gLkEUBTKzyoHuqAycBJwgiEJDAp4D9UYdqgzCTQwN8hAEUWhIwHOAMYZusYxCGThBEIWGBDxH1jYYoVOrUKmjcXCCIAoLHWLmyGd3d2FLezUZ3RMEUXBIwHNkW0cNtnXUFDsMgiDKECqhEARBlCgk4ARBECUKCThBEESJQgJOEARRopCAEwRBlCgk4ARBECUKCThBEESJQgJOEARRojDOVyySz9+LMWYDcCnLT28AMCNjOHJCsWUHxZYdFFt2lHJsazjnltgHCyrgucAYO8w57yt2HPGg2LKDYssOii07VmNsVEIhCIIoUUjACYIgSpRSEvCHih1AEii27KDYsoNiy45VF1vJ1MAJgiCI5ZRSBk4QBEFEQQJOEARRopSEgDPG9jLGzjPG+hlj3yh2PNEwxoYZY6cYY8cZY4eLHMujjDErY+x01GN1jLHXGWMXxbe1CortW4yxcfHaHWeM3Vqk2DoYY79hjJ1ljL3HGPuK+HjRr12S2Ip+7RhjBsbYQcbYCTG2vxIfV8J1SxRb0a+bGIeaMXaMMfaS+Pesrpnia+CMMTWACwA+CGAMwCEA93DOzxQ1MBHG2DCAPs550QcEGGPvB+AE8CPO+SbxsX8AYOecf0f85VfLOf+6QmL7FgAn5/x7hY4nJrYWAC2c86OMMTOAIwA+DOA+FPnaJYnt4yjytWPCHkEj59zJGNMCeBvAVwDcjeJft0Sx7YUyvuf+FEAfgCrO+W3Z/pyWQga+A0A/53yQc+4H8N8A7ixyTIqEc/5bAPaYh+8E8IT4/hMQfvgLToLYFAHnfJJzflR8fxHAWQBtUMC1SxJb0eECTvGvWvEPhzKuW6LYig5jrB3AhwD8MOrhrK5ZKQh4G4DRqL+PQSHfwCIcwC8ZY0cYYw8WO5g4NHHOJwFBDAA0FjmeWL7MGDsplliKUt6JhjHWBWA7gANQ2LWLiQ1QwLUTSwHHAVgBvM45V8x1SxAbUPzr9gMAXwMQjnosq2tWCgIeb927In6TiuzhnF8J4BYAXxJLBUR6/DuAtQC2AZgE8P1iBsMYMwF4BsCfcM4dxYwlljixKeLacc5DnPNtANoB7GCMbSpGHPFIEFtRrxtj7DYAVs75ETm+XikI+BiAjqi/twOYKFIsK+CcT4hvrQCeg1DyURLTYh1VqqdaixxPBM75tPhDFgbwMIp47cQ66TMAnuScPys+rIhrFy82JV07MZ55AG9CqDEr4rpJRMemgOu2B8Ad4tnZfwO4kTH2Y2R5zUpBwA8BWM8Y62aM6QB8AsALRY4JAMAYM4oHS2CMGQH8HoDTyT+r4LwA4F7x/XsBPF/EWJYhfcOK3IUiXTvxwOsRAGc55/8Y9aGiX7tEsSnh2jHGLIyxGvH9CgAfAHAOyrhucWMr9nXjnP8F57ydc94FQct+zTn/NLK9Zpxzxf8BcCuETpQBAP+r2PFExdUD4IT4571ixwbgKQi3hQEIdy4PAKgH8CsAF8W3dQqK7b8AnAJwUvwGbilSbNdCKMudBHBc/HOrEq5dktiKfu0AbAFwTIzhNIC/FB9XwnVLFFvRr1tUjNcDeCmXa6b4NkKCIAgiPqVQQiEIgiDiQAJOEARRopCAEwRBlCgk4ARBECUKCThBEESJQgJOEARRopCAEwRBlCj/H9OIApv6vgQyAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(score);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done after 84\n",
      "Average reward: 28.0\n"
     ]
    }
   ],
   "source": [
    "ag1.Simulate(3, eps=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "ag_r.envs.envs[0].close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## REINFORCE with Baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "\n",
    "S = env.reset()\n",
    "ag_rb = Agent(env)\n",
    "ag_rb.env._max_episode_steps = 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "score2 = []\n",
    "counter = 0\n",
    "mysum = 0\n",
    "interval = 50"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "for i_episode in range(200):\n",
    "    traj = traj = ag_rb.REINFORCE_baseline(alpha=0.0001, alpha_v=0.0001)\n",
    "    counter += 1\n",
    "    mysum += len(traj)\n",
    "    if counter%interval==0:\n",
    "        score2.append(mysum/interval)\n",
    "        mysum = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXzcVbn48c+ZNfuepkm6t+lKW1pKy74jUJaCiqLoBUF5ueGCXgHRi1d/et2vIC6gqIBeBEQBEdla9r0t3bekadomzb5nksx6fn98v9/JTDKTfZmkz/v14tXMN98kh2n6zDPPec45SmuNEEKIqcU20QMQQggx+iS4CyHEFCTBXQghpiAJ7kIIMQVJcBdCiCnIMdEDAMjLy9Nz5syZ6GEIIcSksmXLlgatdX6szyVEcJ8zZw6bN2+e6GEIIcSkopQ6HO9zUpYRQogpSIK7EEJMQRLchRBiChowuCul/qCUqlNK7Yq49hOl1D6l1A6l1D+UUlkRn7tdKVWmlNqvlLporAYuhBAivsFk7n8CLu517QXgBK31CuAAcDuAUmopcA2wzPyaXyul7KM2WiGEEIMyYHDXWr8KNPW69rzWOmA+fBuYYX68Afir1tqrtT4ElAFrR3G8QgghBmE0au43AP82Py4GjkZ8rtK81odS6ial1Gal1Ob6+vpRGIYQQgjLiIK7UuoOIAD8xboU47aYewprre/TWq/RWq/Jz4/Zgy+EEBOmosHD66UNEz2MYRt2cFdKXQdcBlyrezaFrwRmRtw2Azg2/OEJIcTEuO+1cr722LaJHsawDSu4K6UuBm4FrtBad0Z86ingGqWUWyk1FygB3h35MIUQYnx1+4J0+0MTPYxhG3D7AaXUw8A5QJ5SqhK4E6M7xg28oJQCeFtr/Vmt9W6l1KPAHoxyzRe01sGxGrwQQowVbzBEIDiFg7vW+mMxLt/fz/3fB74/kkEJIcRE8wdC+IOT9xhSWaEqhBAx+IIh/KHJm7lLcBdCiBj8wRBaQzA0ObN3Ce5CCBGDL2Bk7f5JWneX4C6EEDH4zHq7BHchhJhCejJ3KcsIIcSUYWXsk7UdUoK7EELEYGXuPgnuQggxdfRk7lKWEUKIKcPK3AOTtNddgrsQQsRglWN8AcnchRBiypDMXQghpiCr5i597kIIMUUEgiGsXQekz10IIaaIyIAumbsQQkwRVr0dpBVSCCGmjMiFS5K5CyHEFBEd3CVzF0KIKcEfWZaRVkghhJgaIjP3yPr7ZCLBXQgheomaUJWTmIQQYmqQCVUhhJiCImvuMqEqhBBTRGTmLod1CCHEFOGXsowQQkw9PinLCCHE1OOTvWWEEGLqkVZIIYSYgvyyiEkIIaYeK6ArNYW3H1BK/UEpVaeU2hVxLUcp9YJSqtT8M9u8rpRSdyulypRSO5RSq8dy8EIIMRaszD3FaZ/SW/7+Cbi417XbgI1a6xJgo/kY4BKgxPzvJuA3ozNMIYQYP14zc09xO6J63uPZX9NOtz841sMakgGDu9b6VaCp1+UNwAPmxw8AV0Zcf1Ab3gaylFKFozVYIYQYD+HM3TVw5t7tD3L5Pa/z2JbK8RjaoA235l6gta4GMP+cZl4vBo5G3FdpXutDKXWTUmqzUmpzfX39MIchhBAj960ndvLo5p7Q5QuEcNoVLrttwFZIrz+ELxCivds/1sMcktGeUFUxrsV82dNa36e1XqO1XpOfnz/KwxBCiMH75/ZqXi9tCD82grsNh9024CImvznhGkqwlsnhBvdaq9xi/llnXq8EZkbcNwM4NvzhCSHE2AqGNG3dfroiaub+YAiXw4bTrgbM3INmUE+0tU7DDe5PAdeZH18HPBlx/T/MrplTgFarfCOEEImotcuP1kRNiPqCIVx2G067bcBWSCv4h3RiZe6OgW5QSj0MnAPkKaUqgTuBHwKPKqVuBI4AV5u3PwOsB8qATuBTYzBmIYQYNc2dPgA6fRHBPaCNsoxNDViWsSZcJ11w11p/LM6nzo9xrwa+MNJBCSHEeGnpNCZCu3zRmbvbYcPlsNHhDfT79YFwWSaxgrusUBVCHNdazMw9quZuTaja1ICtkFbZJphgmbsEdyHEca05TuZuTKgO3AoZLstI5i6EEIkjZuYeNPrcBxXcp1i3jBBCTAnWhGpk5u4NGJm7wz6YCdXE7JaR4C6EOK5ZE6q+YCgcqI3M3WyFHCAl9ydot4wEdyHEcc0K7tBTmvEFjG4Zp13hH6CWHpRuGSGESDxWWQZ6gntk5j5QzT28/YBk7kIIkTiaIzN3X0/m7nLYcNhsA7dCBiVzF0KIhNPS6SPNbazn7MncjRWqTocacD/3YDhzH9txDpUEdyHEca2500dhZhLQk7lb3TJO2xAmVBMsuktwF0Ict7r9Qbr9IQqzkoGe4O6P2DgspPsvucgKVSGESDBWp0yRlbn7e9Xc7cYRFf1NqkrNXQghEozVKVOYaWTunb7IbhmF0wzugX4zd+lzF0KIhFDf7uUPrx+iyWME96Ksnsw9FNIEQhqX3Y7TboRIf6C/zN06iWmMBz1EEtyFEMedv22p5LtP7+Gf242D4orMmnu3PxjujnE6FA4ruPcTucN7y0jmLoQQE2tfTRsAf99aBRDulun09QR3l92GK1xz76csI90yQgiRGPZWG8HdCuRWzb3LFwyXYKxFTEC/7ZB+6ZYRQoiJ5w0EOVjv4YIlBQAkO+0ku+wkOW10+aMzd6fDLMv0E9yDCdotM+Axe0IIMZWU1XUQDGmuXFVEfXs3DR3GpGqy025m7kaQdtptOG0Dl2X8CdotI8FdCHFc2VfdDsDi6Rn85OqV1Ld7AUhxOczM3WiHNPrcrbJMfzX3xOyWkeAuhDiu7Ktpw+2wMSc3BYfdxsKCdACjLOML4ovM3M0J1f72lwlKt4wQQky8fTXtLCxID2fllmSXParm7jbPUIUBJlSlW0YIISbe3up2Fk9P73M9xemg0xcIT55a+7nDAK2Q0i0jhBATq6HDS0OHl8WFGX0+l+Sy0+UP4YtshbT63AexiEkydyGEmCA1rd0AzMxO7vO5ZKeN7ohFTE67wjWU7QcSK7bLhKoQ4vjR2mXsApmZ7OzzuRSXg05/ICpzt9sGsXGY9LkLIcTEsrb4zUzpG9yTnHa6fKFwzd1lt2GzDWLLX+lzF0KIiWVl7lnJrj6fS3HZ6fIFaO8OAEb3jFVqH9SEaoJl7lJzF0IcN/oryyQ7jVbIg3UduB02CjOTwxOqg2mFnFLdMkqpryqldiuldimlHlZKJSml5iql3lFKlSqlHlFK9X2JFEKICdDa5cdlt5Hk7Bv6kl12Qhr2VLcxPz8Nu01FtEIOvIhpynTLKKWKgS8Ba7TWJwB24BrgR8D/aq1LgGbgxtEYqBBCjFRrl4+MZCdKqT6fS3baAdhZ1UpJQRpAeIVqv3vLJGi3zEjLMg4gWSnlAFKAauA84G/m5x8ArhzhzxBCiFHR2uUnK8ZkKhiZO0B7d4AF+VZwHzhzT9RumWEHd611FfBT4AhGUG8FtgAtWuuAeVslUBzr65VSNymlNiulNtfX1w93GEIIEfaR377FI+8difv51i5/zHo7GBOqFitzdwziDNVggnbLjKQskw1sAOYCRUAqcEmMW2P+H2ut79Nar9Far8nPzx/uMIQQAoBmj493K5rYcrg57j39BfckZ09wXzDN2J7AaRs4c/dPwW6ZC4BDWut6rbUf+DtwGpBllmkAZgDHRjhGIYQYUHlDB0D40OtY+gvuVs3dYVPMzk0BwGZT2G1qUGWZ/jL3lk4fNz24meZ+xjbaRhLcjwCnKKVSlDE7cT6wB3gJ+LB5z3XAkyMbohBCDKy83gNAYz8BtKVz4LLM3LzUcK0djGDf737u4bJM/LHtOdbG83tq2XWsNf5No2wkNfd3MCZOtwI7ze91H3ArcItSqgzIBe4fhXEKIUS/yhuM4B4vcw+GNO3dgQHLMgumpUVdd9lt/e7nbvXA91eWsU5r6u8dwGgb0QpVrfWdwJ29LpcDa0fyfYUQYqjK682yTEfs4N7eHX8BE/R0y5T0Cu4Oe/+Z+2D63INmXd7XzwZko01WqAohpgSrLNPuDeANBPt8vr/VqQDTM5IomZbGWQujGzycdtvgJlT7qblbLw7ecQzusreMEGLSC4Y0hxs7yUhy0NYdoNnjZ3qmPeoea9OweH3uqW4HL9xydp/rRnAf2a6QgXBZZvw6aiRzF0JMelXNXfiCIU6anQ1Ao8fb556BMvd4nHYV3hwslsHsCmndI2UZIYQYgoNmG+SaOTlA7EnV4QZ3l8NGt79vmccymMM6rHt8McpFY0WCuxBiUnlpXx11bd1R1w6Z9fY1Zube5PFRWtvOd57aHZ7oHG5wT3Y56PT1F9wHX5bpr+tmtElwF0IkrE5fIOpxaW07n/rTe/z5negtBsobOshIclBSYKwsbezw8cS2Kv70ZgX1HUaJxgruGUMM7qkuO139BfeIoB6vYyYoNXchhDDsPtbK8u88T1ldR/jaH944BIDHGx30y+s9zMtPIyvZiU0ZmfvBOiObtyZSW7v8JDltUdsMDEaKy95/5h5Rj4/XMWOVZcazW0aCuxAiIe2vaScY0lSYi5MaO7w8vrUKgK5eNfBDDR7m5aVisymyU1w0enwcNPverYy9tZ/Vqf0xyjKBmJ/TWuMPahzmcXzxSjMyoSqEEKYas67e1GlMjv757SP4AiGSnNETnJ2+ANWt3czLTwUgJ9VFfXs3FY1W5m58fX/7yvQntZ/M3YrlbocRSuM1zFh1+fFcoSrBXQiRkOrajFq5FZxfOVDHSbOzKc5KxuvvCZLW4qV55h7sOakuth1tDde3W8zMvaXLF/Ps1IEk91Nzt4K12yz1xC3LSOYuhBCGmlYjc282a+aNHh/FWckkOe1Rmfshs2wzN8/I3HPTXDR09PS5t1llma7AkCdTway5+4PoGIHbCtouc6OxeGUZ2X5ACCFMte1mcDd71hs7fOSmuUgyD7K2lNd7UKonuOek9mTnSvVMqLYNsyyT4nIQDOmYk6FB892ByyzLxOuW8U9AWUa2HxBCJKTacObuo9sfpMMbIC/NTbLTHjXBWd7QQVFmcrgLJifVDUB+uptAMERLlw+tNY0eL9lxth7oj7UVcJcv2KfTxtpXxqq5xyvLWBm9V2ruQojjWSikqWs3SivNnf7witPcVJc5odoTJA81eMKTqdY9APPzU8lKcdHaFaDDG6DbH2JahnvIY7GCe2eMVaqBwWbuUpYRQgijvm7Vs1s6fTSa2/jmprlxO+10m8v4tdZGj3teT3DPCQf3NDKTnbR0+sIvFNPSk4Y8lhSXUeDo9PZthwz0ytzjLVINSreMEEJArdkGmZPqosnjp8HcCCwn1UWSw0632b1S3+6lwxsId8pY90BPcG/r8lNvBvf89BFk7jE6Znpn7tItI4QQ/bCC++Lp6VGZe16ai2SXjW4zSB4Mt0H2ZO4LC9KZnZvCaQtyyUpx0tLlj8jchx7ck/sL7mbm7nIY98QrywSkLCOEOB4dqG3n0feOhh/Xmj3ui6dnEAhpjpgLknLT3Ebmbta/e7dBgpGdv/Kf57J4egZZyU5aOkeWuadaZZkYq1StjDw8oTrg3jIS3IUQx5GH3jrMNx7fEQ7CNW3dKAULC4xyS1l9B26HjVSXPdznrrWmzmyXLMxMjvl9M5OdtHX7qW3rxmW3DbMVcuCyzEDdMv4JOIlJgrsQYsJZQf3VA/UA1LV1k5fmJi/NyLTL6jrITXWhlCLZZSekje1zPd4AyU47dnNvl94yU1xobZyvmp/uRqnY9/UnOaIVsjcrE3eFtx/oP3OXLX+FEMcVa1vel83gXtPWTUGGm+xUI9M+1OAh1wz0Vpbc7Q/h8QVJdcff5THLzNRL6zrIG0ZJBnrKMp4YZZlguCxjbj8QJ3bLhKoQ4rhkbRfwWmk9wZCmts3L9IwkslOMzhd/UJObZnxsLSTy+oN4vAFS3fHXYlplmCNNncOaTIX+J1T9vcsy8SZUzagvNXchxHGlvt0I5i2dfrZXtlDb1s20iOAOkGuuPLWCe7c/hMcbDPehx2Idhq318CZTwQjcNhW7LNPTLWP1uUsrpBBCAMbBG52+IBtWFWFT8JW/bqPJ42NmdgoZyU6sMrmVuSebwb3LzNzT+ivLRGw3MNzMXSlFqssRsywz1G4ZCe5CiOOGVZIpmZbOGSX5eLwBPnfOfK47bTZ2mwrXza1tBZKcVs09iMcX6Ddzj9wFcriZO8Tf9nfw3TJWWWb8jtmTjcOEEBPKCu55aS7+eP3JAFHdL9kpLpo7/eEJ1Z6yjJG5z8xOifu9I1sf89OGH9zjHbUXGEa3jNZ6WF07QyWZuxBiQkUuMLLbVJ+2Rqu00jOhambuAaPm3l+3jNthD/epT8sY+r4ylpQ4R+313c899tcHIjL28WqHlOAuhJhQ9ebWAvEya2tStacs09N3PlBZBnqy95GUZeJm7tbGYdZJTANsPwDjV5oZUXBXSmUppf6mlNqnlNqrlDpVKZWjlHpBKVVq/pk9WoMVQkw99e1elIo+ZCNStnm9d1nGG7AmVAcX3PPShn7EniU5TnC3ArWVucfrlokM+uM1qTrSzP0u4Fmt9WJgJbAXuA3YqLUuATaaj4UQIqaGDi85KS4c9tjhyDpgo3fm3uzxEdKQ0k9ZBoyyTlaKM7zQaDhS45RlwouYnP13y0Rm6+MV3Ic9oaqUygDOAq4H0Fr7AJ9SagNwjnnbA8DLwK0jGaQQYnJo7/bT6QtSMIT6dn27t9+SyeUri0h22sNBPcmcvGw0D/AYKHOfkZ0y4j1dBppQtV44BpO5j9dCppF0y8wD6oE/KqVWAluALwMFWutqAK11tVJq2siHKYSYDH763H5eK2tg09fOGfTXNHR4w3vIxLJiRhYrZmSFH1srRq3gnjpAzf3Oy5eOuM4drxXS3/skpnitkBE19/HaPGwkZRkHsBr4jdZ6FeBhCCUYpdRNSqnNSqnN9fX1IxiGECJRHG3uoqLBE85oB2OgzL23JDNLbjRbKPvrlgFIT3LGrecPVqrbETNzDw6yWyYY0uEun8lQc68EKrXW75iP/4YR7GuVUoUA5p91sb5Ya32f1nqN1npNfn7+CIYhhEgUTWYdvNZsbxyI1trM3AcffG02hctuCx/g0d/eMqMl2Wmnyx/scxhH+IDsAWrugaAOd/WMV1lm2MFda10DHFVKLTIvnQ/sAZ4CrjOvXQc8OaIRCiEmjeZOI+Aea+ka1P3WwdVDbVN0O23hssxArZCjweqV7+p1SHZgkN0ygVAovG3CePW5j/RZuRn4i1LKBZQDn8J4wXhUKXUjcAS4eoQ/QwgxSTR5Bh/cn3i/Klw/H2pwT3bawytbB5pQHQ0pbus0pmDUO4Wh7C2TkWQG90TvlgHQWm8D1sT41Pkj+b5CiMnHHwzR3m20Cx5r6e733rK6Dr7yyLbw4/4mVGNJctrD56JaWfVYSnFa2/4GgJ6xBoIhHDaFzVxVG3dCNajD45QVqkKIScUqyQBUt/afub9d3gjApSsKyU11sbAgfUg/y5qchHHK3OPs6R4IaRx2hU31H9yDIR1+lzIpMnchhLC0dPrDH1tlmbr2bvJS3eHM1vLOoSYKMtzc87FVwzv6ztmTrY/HhGpkWSZSIKhx2GzYzf+H+CcxhcJzA5OhW0YIIcKawhOcdqpauqlu7eKMH73EzQ+/H9UaqbXm7fJGTpmXO+zdEa29XJx2Fe4xH0s9mXv0KtVAKGRk7uYQenfThO8Ljn/mLsFdCDEqms3gvrQwg+rWLt462IgvEOJfO6v5+mPb8QaMrLe8wUN9u5dT5uUO+2dZq1XHI2uHnncKvTN3v5W5m+9MYu3nrrUmENLhun3Ct0IKIUSkJrPmvqwog5ZOPy/trycz2cnXP7CQJ7Yd49K7X+fdQ028U94EwLq5OcP+WdYWBAOtTh0t1otI71WqwVAIp12FyzKxau5WB814T6hKzV0IMSy9D52wMvdlRZkAvLCnhjMW5PPF80pYVpzJt/6xi4/c+xbpbgfT0t3MzUsd9s+2ShwDrU4dLVZg7n3UXiCosdtU+HmIVZax2iWtur2UZYQQCe2Ke97gFy8eCD9u8vhJddmZnWucjNTtD4Wz83MXTeP5r57FHeuXkJbkYP3ywhGdRmRtQTAeC5gAMpKMnSnbuqKDuz+kcdojyjIxgns4c59ki5iEEMehJo+PnVWtUQdQN3f6yE51UZSVHL62bl5P6SXV7eAzZ83jM2fNG/HPt1ohx6MNEox3CklOW1S7JxhlGYetpywTa38yaxWrtEIKIRLerqpWAA43doavNXl85KS6mJ6ZhFJGPXxpYcaY/PyeCdXxKcsA5KS4wh1BFr9ZlumvW8Y6hclpt+GwKQnuQojEtdMM7pXNnfgCIVwOI6vNTnHhtNuYnpHEwoL0uAdwjFQ4uI9TWQaME6GaewX3QDAUVZaJNaFq1dwdZtvmZNjPXQhxnLIy95CGqpYu5ual0uTxMT8/DYC7rlkVPtB6LIx3KyQYxwA29SrL9F6hGqsVMhzcbQqn3SYTqkKIxLXrWCuFmcZpSxWNHsBYoWodZr12bk440I8Fq+Y+0BF7oyk7JVbmro29ZfrplgkGreBuw+Wwyd4yQojE1NLp42hTF+uXFwJwuMGDNxCkwxsIn3c61qzMPW0cyzI5qX1r7oFQKHoRU4y4be357rAb+9D7AiM7FWqwJLgLIYZk97E2AM5ZlE+qy87hps7wvjLZIzzxaLCsFaMp41iWyU5x0dYdiKqZ95RljMexyjJWK6TdpiRzF0IkLmsy9YSiTGbnpnK4sTOc0Y70OLvB6mmFHMdumVTjXUlkO6RVllHKCPCxyjLWi4HDZjMz977H9Y0FCe5CiCHZUdnCjOxkslNdzM5NoaLRE65FWzX3seaegAlV611Js6dn90tvIBjeuMxuU/1uP+CwWd0yUpYRQiSYUEjz1sFG1s01Nv2anZvK0aZOatuNwznGK3NPnoBWyBzzhSuy7t7tD4Xr/0qp/rtl7Aqnffz63CW4CyEGbW9NG82dfk5fYAT3Obkp+IOa/3lmH7mpLmbmJA/wHUbH6lnZ3HLhQk6dP/ydJYcqnLl3Rgb3YHgrBLtSsRcx9e6WkeAuhEg0b5Q1AHD6gjzAyNzBaIO895MnjdteLy6HjS+dXxLOmseD9a4kOnMPhuv/dpuK2S1jrVA1JlTteGURkxBiPLR2+klx23EOYjXpG2WNzM9PpSDD6HFfUphOcVYyX79oIWvmDH8L38nA2kcnstfdG+gpy9hUnBWqZubutCtcdoVfMnchxFjTWnP+z1/hvlfLB7zXFwjx7qEmzjCzdoCsFBdv3HYeV62aMZbDTAhuh500tyO8SlVrjTcQCk/uDjShKq2QQohxU9/hpaHDy7ajLQPeu+1oC13+IKdFBPfjTXaqM5y5e80M3CrL2JSKueWvNaHqtFutkBLchRBjrKrZOMi6rK5jwHtfL2vAphjR8XiTXU6KiyZzwVa33+hXtyZUbXEyd+v8WLu5t4wcsyeEGHNVLUZwP9zoCZ9xGs+bZQ0sL84kM3l8thhIRJE7Q3b7rcy9p1umv8zd6nOXzF0IMeaszD2koaKhZ2/2tm5/VFeIxxtg29GW47okA9F7uocz90F2yzjs0gophBgnVuYO0aWZW/+2g2t//0748buHmgiEdNRk6vEoO9UV7nPvDljB3SrL9N8t47CZG4dJWUYIMdaqmo292JWKDu5bDjezt7qNo01GNv9GWQMuh42TZmdP1FATQk6qi05fkG5/MKIs0zOh2u/2A/aebhkd477RJsFdiONYVUsX8/PTKM5KpqzeCO717V7q2r0AvLS/DjAmU9fMzh7XRUOJyNo7p7nT12dCNV7N3R/ZCmm3oTXjsr+MBHchjlNaayqbu5iRnUzJtLRw5r6n2tjS125TbNpXR01rN/tq2sOrUo9n2eGFTP5wcHc7+++WCUbsCmm9OI5HaWbEwV0pZVdKva+Uetp8PFcp9Y5SqlQp9YhSanx2EhJCDElbV4AOb4DirGQWTEujvL6DYEizx9yvfcOJRbx1sJHb/r4Dl8PGpebhHMczaxfKTl+gT1lmwG4Zu8Jt3mu9MIyl0cjcvwzsjXj8I+B/tdYlQDNw4yj8DCHEKKtsMerpxdlGcPcGQlQ1d7H7WCszspO58sRivIEQL++v5xsXLWJOXuoEj3jipZr7x3d4A+HW0aSIzD12t0zPhKrb3B7YOw4dMyMK7kqpGcClwO/Nxwo4D/ibecsDwJUj+RlCiLFhtUHOyE5m0fQMwKit76luY2lhBmvn5pDudrBubg43nD53IoeaMHoy92BEK2TP3jKxJkp79nPvKcuMR+Y+0o3DfgF8A0g3H+cCLVrrgPm4EiiO9YVKqZuAmwBmzZo1wmEIIYbKaoMszkomJ9XFyplZ3LXxAHXtXjasLCbJaefJL57OtIwkbNY5csc5a/94jzeiLBNxWEes/dx7TmKKyNz9CZy5K6UuA+q01lsiL8e4Nea0sNb6Pq31Gq31mvz8/OEOQwgxTFXNXSQ5beSkulBKcdvFi6lt86I1LC0yMvl5+WmkjeNpR4kuxWVk3rEz99g192BIo5RRtrEmXwdaDTwaRlKWOR24QilVAfwVoxzzCyBLKWX9NswAjo1ohEKIUdfpC7C9soXirGSMaiqcOj+XcxYZiZYV3EU0qyzjiZpQ7X9XyEBI47QZodbK3LsTOXPXWt+utZ6htZ4DXANs0lpfC7wEfNi87TrgyRGPUggxLAdq2/tsCvbS/jrO++krvFfRzFWroqumP7hqOT/84HKKs8bnRKXJxu2wYbcpoywTCOK0K+xmySput0wwFL7H7ZgcmXs8twK3KKXKMGrw94/BzxBCDMLXH9vO9X98l4C5KvK3rxzkhj+9R1aKk8c/dypfPK8k6v6irGSuWStzYPEopUhx2fF4g1FH7Bmfg1CcbhmH3QjuSc7xy9xHpZimtX4ZeNn8uBxYOxrfVwgxfFpryuo66PQF+dfOajzeID/89z4uXVHITz68YtyOxJtqUl0OOn0B7DZbuIYORlkm1qZggaDGMQGZu/ztCrfu0ToAAB3ASURBVDFF1bZ56fQZQeSuF0upaevmjAV53POxVeE6uxi6FLcdjy+I267DmTjE75YJhDR2s+Zu3Z/Q3TJCiMRW3mDU2q88sYjyBg82pfjRh1dIYB+hNLeDTrPmbk2QgrlxWMxumRBOu2TuQohRcqjBA8BXL1yINxDiipVFMlE6Cqyau1IqaiO1uJl7UPdMqDrHb4WqBHchpqjyeg9JThszs1P4zSdOmujhTBmpLgc1bd047NHB3ehz73t/INRTc7cmYCfL3jJCiAR0qMHDnNxUWV06ylLcjvAipsiae7ztBwKhEA67cZ/TrlBqEuwtI4RIXIcaPMzLl82+Rlua2x7efiCyFdI4Zi92WcbK3JVSJDnskrkLIYbHHwxxpKmTeXlpEz2UKSfFZWbugWB0WSZOzT0Y0ecORt1dMnchxLAcaeokGNLMlW16R12qy25sP+ALhidIwVihGqtbxh/RCglG3V1aIYUQw3Ko3uiUkbLM6EtxO9Aamjv9g+qWCYZC4bIMGJl79yTdfkAIMc483gBn/ngTj2+pBHp63CVzH33W5mFdg9x+wB9RcwdjfxrJ3IUQg7JpXx1Hm7q4a2Mp/mCIx7dUUTItjawUOeVytKW6egJ6Uu+yzCBq7klOu2TuQojBeWZnNXab4khTJ199ZBv7a9v54nkLJnpYU1Lknjx9yjJxzlB1RNTcJXMXYorq8Ab42qPbwytIh6u1y8/uY610+gK8tL+Oa06eyZzcFJ7eUc2CaWlctqJolEYsIlnnqEJ05m6Lt597MLrmnuS0y/YDQkxFD711mMe3VlLX3s2DN6wd9l4vX3t0Gxv31XHJCdPp9oe4bEURS4syuOMfu/jS+SXhJe9idMXN3Ps5icneq+be2CHbDwgxpXT7g9z/+iHSkxy8VtpgHJyxuGDI32fPsTZe3FtHXpqbZ3bWkJfmYu3cHNbNzaFkWjonz8keg9ELIOrYwcgJVZsiZnD3B0M47ZFlmfHJ3KUsI8Q4emxLJQ0dXn597Wrm5aXy/57eSyDWhiS91LR209rpDz/+1ctlpLkdPPuVM/nU6XP46oULsdsUNpti7dwc2flxDKVETKi6e5VlYlRl+mbuTtvkOaxDCDE4f3zjECfOzOKMBXl8/twFfP2x7Ryo7ej3zNJQSPOh37xJIBTioRvXUdXcxTM7q/ns2fPJS3Nz5+XLxvH/QKS6+ynLxNnPPWqFqsMuu0IKMZW0d/spr/fwnxctQinFqllZAOw+1srSogz++u4RZuemcur83Kiv23KkmaqWLlx2G5f98nV8gRBz81L59BlzJ+J/47iXEtUKObS9ZYyvsUlZRkw9VS1dfPg3b1LT2j3RQxl3B81VoyXTjP1e5uSmkuy0s6e6jW5/kP96ajf/+bft+HuVaf61oxq3w8ZTN5/Ourk53HrxYp79ypnkprnH/f9BGBOiPVv4DqJbptf2A27ZfkBMRS/uqWXz4WbePNgw0UMZd2V1xqrRBWZwt9sUSwrT2X2sjR2VrfgCISqbu3hq27Hw1wRDmmd2VnPuomksnp7BQzeu43PnzA+f6CPGn3VINtBrP/fYE6qBiJOYwHhx8AVDMfehGU0S3MW4eq+iCYD9te2j9j2PtXRxxo82hb93oiqr68BltzErJyV8bWlRBnuPtfF2eSNgbBfwq5fLuPnh91n2X8/ypYffp67dy6UrCidq2CIGq+7eu+Ye0n33dA8GoydUra8Z67q7BPch6v2WeSrSWrOzsnVMMosth5sBOFAzesH9j28corK5i41760b0fbTWvHuoiS/8ZSu/feXgKI2uR1ldB3PzUsMHNwAsK8qk3RvgH+9XsaggnVsuXEh5vYfndtVw8twcntlVTZLTxnmLp436eMTw9WTu0WUZoE/HTCCke7VCWkftjW3dXSZUgTfLGghqzZkl+f3eV1bXwfq7X+P/Pr2ONXNyxml040trzff/tZffv36I7191Ateum93v/e9VNPHy/jq+eG4Jya7+SwVVLV1Ut3bjsCkO1HbEvMcbCA6p5NDe7eev7x4FYEdlCwCbK5qoaOzkwyfN6PdrvYEgP/jXXjasKmb1rGz+59/7uO/VcmwKnt1dwwVLCsIllN5f9+kHNnPp8kKuWTurz+d3VbXywp5ayuo6+OalS8LnlpbVtbOsKDPq3qWFRpfMoQYPnzxlNuuXF9LW7efUebnMy09jV1UrHm8gqkNDTLy0OJk7QFBrbPRk6oFQSDL3ifLdp/fw7Sd2DXjf37dW4guE2Ha0ZRxGNTH++597+P3rh7DbFK8dGLgu/uNn9/Grlw7ywd+8ydGmzvD1tm5/n3s3m2WTi5ZNp6qli/Ze92zaV8uq777AG2WDr8c/8t5R2r0BTpyZFX638aNn93H733dE9YXH8vvXDvHAW4f59AOb+fPbh7nv1XI+umYmr916HilOO//zzN6YX/fE+1W8VtrAt5/cFX5BsbxX0cSVv3qDX24q5d+7qrlnUylgLF460tTJ/F4vFoump4f/4a+dm4Pdprh23Wzm5Rv3nVCcybp50d0zYuJZq1SjFjGZf4+96+6RZ6hCT+Y+1qcxHffBvdsfpLSug4rGTurbvXHv01rzzx3GRFdkEJtMalq7+93PpLy+gz+9WcEnTpnFVauKeftQY5/SjDcQ5PN/2cLWI80ca+nivYpmLlxaQFVzJzc9tIVQSPOvHdWs+u4L4RfB6tYuKps72VzRTKrLzuUrjT1PIrP3+nYv//nYDjp9Qe57tXxQ/z+bK5r49csHWTsnh4+vnUW7N8C2yha2HG7GH9Q8v6cmfG8opHn/SHO4HlrV0sUvN5Vyyrwc/MEQ33piF4unp/PfG5ZRnJXMF89bwMZ9dX1eaIIhzb2vlLN4ejr5aW5ufvj98ItUbVs3n//LVmbmpPDeHRfwsbWzeHxLFXVt3VQ0eghp+rwTSHLaWWAG8rVzp+a7wanI2l8mahGTmblHdsx4vAG0js7wra+RzH2MHahtD7/Sbj3SHPe+bUdbONrUBRin3Iw144zG+K/svkCIPcfahvTq/5VH3ueT978T8xBfgD+9WYHLbuPL5y/ktPm5tHT62VvTFnXPtiMtPLOzhm8/sYt/bjde7O5Yv4TvXXkCe6vbePi9I/y/f+0hGNI8+GYFvkCIq3/7Fuf+9GWe2FbFqlnZLDMX7BwwJ1W11tz2+A7avQGuWFnEKwfqqWjw0Nrpp7HDeME92tTJWT9+KfyC8a8d1Xzsd2+TnuTgBx9czoqZRrnjnk1lhLRRC316R3V43A+8VcFVv36TuzeW0eULctvjOwD42UdO5NfXruaE4gzu/tiq8D/C606bQ1FmEj9/4UDU8/Xc7hrKGzzcfF4Jd31sFUebOrnjH7vo9AW46aEteLwBfvuJk8hNc3PTWfMIhELc/8YhSs0XspIYZZ5183JYWphBQUbSoP8uxcRKcTlQqicLB7DK6pGZe2mvDinoyfbHOnM/7gt5e471BK8th5u5aNn0mPc9tf0YLruNtXNzODzC4P7QWxU8tqWS2y5ZzGnz8/p8vtsf5PJ7XsfrD3HvJ09iYUE6tW3dzDS7LO56sZTfvFJGtz/E58+ZzzcuXhzz5/gCIf69q5oPLJ1OW7efdw41obXRqbJ4evSKyNZOP49truSKE4vIT3eHF9K8dbAxqk787iGjtLL7WBuHGjwsL85kTl4qs3JSuP/1Q3zriV1oDatnZfH0zmrmT0ujsrmLsxbm81ppPWeW5FGclUyKy85+c1L14XePsnFfHf912VIuW1HIMzur+faTu9hV1Uqq28HGr53NH944xJGmTh557wgrZ2Tyk+f2sbAgnf/79ClkpjgJhjQpLjub9tWRleLkI2tm8ofXD9Hs8ZGV4uThd4/gsCn+98UD/G3rUSqbu/j+lcspzkqmOCu5z3xLktPO586Zz7ef3M1bBxs5bUEerV1+fvLcfubmpXLxCdOx2xRfvWAhP3vhALuOtVLR4OG3nziJRdPTAZidm8r65YU8+OZhlhZlYFOxD8/49mVLCQTHti1OjK5UtwO3wxa1zUNP5t5z334zOVps/k6AZO7jZvexNtLcDlbPygrXhMEoUfzqpTK01oTMXuNzFuWzrCiDyqauQXeSNHt8dHgDUdce31rFjspWPv67d7g3RlfGXRtLKa/34A2E+OCv32T5d57jzB+/xCsH6qlv93LPS6WsmZ3DsqIMnt1tlB6qWrr489uHo7KG371Wzpf/uo1fbirlmZ3V4Vn8yK4SrTVbjzTz7Sd30eUPcsPpxqrHwsxk5ual8tbBRrp8QVq7jNLDuxVNlExLY1lRBp2+IJevNFr0bDbF7ZcsQWu4dHkhP/jgcnyBED95bj+rZ2XxwKdO5p1vns+NZ8zFZlOUFKRzoLadQw0evvf0Hs5YkMf1p81hWkYSF58wnddKG0h1O6hs7uK+V8p5bLNxwtBzu2vZeqSFisZOrjt1DpkpTsDoGT/BfBE6e2E+V6wsIhDSPLu7hm1HWzhQ28Gdly/lzJI8Wjx+7r9uDR9f13cyNNLVa2YyLd3NLzaW4vEG+Ooj2zja1MmPPrQiXCf//LkLOHVeLuX1Hv7flcv5QK/k4Pb1S1gzJ5sth5uZk5sa9fbc4rTbBpyMFonllHk5nL8kesM363ciMjbsr+kgyWljZnZP+2t4QnWMFzJN6sx9f0079756kB9ctTzmP5reqlq6+Nyft5Cb6uLylUVceWKxsfS7MINVs7L44xsVdPuNE81/uamMf7xfxbmLpqHR1LZ5uXBpAb5gCF8wRE1bN0VmF0SkPcfauGvjAX7+kRNxO4zl4m3dfj51+ly+cO58giHNrqpWPnX6HPbXtPPHNyr4zJnzwpMxe461cd+r5Vx90gxuu2QxP3vhAMlOO//eWc0vXjzAeYum4Q9q/nvDMl4vbeDOp3ZzsL6Dnzy7n2d31/D+kRZ+/OEV1LR188tNpTjtij++UcHs3BQWT0/H5bCxaV8dXzjXOMjhh8/u495XyrHbFJ84ZVbUHienzMvl8a2VrP3Bi6S6jAx66+FmrlpdzOUrivjmP3ay4cTi8P2nzs/l4c+cwvIZmaS5HayZnc3mw83ccqGx3H5aek/ZYVFBGv94v4qLfvEqyU47P716Zfg5uH39Ek6cmcUnTpnNDX96j5+9cACAL5w7n1+9dJD/enIXLruNi06IDqQrZmTybkUT5y6axrKiDBYVpPODZ/aypDCDFJedq1bP4OPrZtPlD0bt7BdPktPOZ8+ez3ef3sOyO58D4HtXnhBVG7fbFPf+x0nsr2nn5BgdVMVZyTx04zq2H23B5Tjuc6kpY8OJxVG/+9AT3CP3lzlQ287CgvTw7zaM34TqsIO7Umom8CAwHQgB92mt71JK5QCPAHOACuAjWuv4xewRaOjw8vetVZxQlMkNA+yzUdvWzcd/9zZNHT6aPD5ueXQ7rV1+9la389GTZ3LS7GzufbWcXVWtLCvK5HkzI960rza8dPjshfnhxTdHmjpjBvcH36rgud21PL61ksLMZKpaulg5M4u7N5bidthYNTOLQEhzVkk+y4szueXR7eysamXlTGOfkR8+u4/MZCd3XLqErBQXP7hqOWC8nf/WE7vYX9POGQvymJ+fhtth486ndvPAmxU8v6eGkmlpPL61krL6DnzmW74/XH8y1//xPfbVtPO1CxcSCGnu3lRKk8fHxr213PtKOdecPJPbL1kSzoIt65dP58ltVayelc0rB+r53tN78PiCnDwnh3Xzctn4tXP6/P9H7ovyzUuX8EZpA6cv6NvtsX55IWV1Hayalc2HVs9gemZP4C/OSubTZ84D4BsXL+bKX73BSbOzufm8Ev70RgW7j7Vx0bICMpOjx/uBZdN5+UA95yzKRynFHz51Mv9x/zu8e6iJq0+aEQ7ogwnslutOm0NumotjLd0UZiax4cS+B2BkJDljBvZI1t+vmLrCZZmIzH1fTTvnLoou+VmtvmNdlhlJ5h4Avqa13qqUSge2KKVeAK4HNmqtf6iUug24Dbh15EPt6/QFeZw2P5dfvVTGZSsLufPJ3SwtzODm80ui7guGNJ9+YDMN7V4e+vQ6Vs3M4hP3v8P//HsfvkCIZUUZrJ5t7H+9aV8dde1ePD4ju9u4r44kh53F09OZlpEU3qrzSGMnyU47pXUdXLGyCJfDRiAY4vk9tYAxOTknN5X8dDd/++ypXPv7d3h8ayX+YAilYPXsbLQ2Vq49v6eGlTOz2FHZwqsH6vnGxYv6nH159ZoZ3LOpjJq2bj5xitF7PiM7hSWFGTz41mHsNsUDN6zl2V01PLmtyug+uWgxZ5bk86HVxTy6uZL1KwrxeAPctbGUj//ubUrrOjh9QS7fu/KEqEUWljNL8tnz3YvRWvPB37zJX98z+skH29WxelY2q2fF3lf8nEXTOGfRwAtzTpyZxY8/vIITijJJcto5b0kB/9x+jCt7ZU3WuF685ezw4+KsZP722dO456Uyrj9tzqDG3JvdpvpkaELEYlPRmXtjh5eGDm94DsZiLXxK2FZIrXW11nqr+XE7sBcoBjYAD5i3PQBcOdJB9ufrFy2i0ePjwp+/yr931fDrlw/S1u3ncKOHz/9lCwdq2/nz24fZWdXKDz+0gtWzslFK8Z3Ll4VfYZcVZZKX5ubS5YXc92o5v9xURl6amxvOmMu2oy28V9HE2earb2FWEnab4nCTh689tp2vP7adC//3Fd4ub+S9imaaPD4uWlZAeb2HTfvq+PBJM3DabXxodTHl9R4efvcIi6dnkJnsJCvFxbq5OTy/23hB+NVLZWQkOfjkKX0XDrkddm5fv5izFuZzwZKeoHih+fHFy6ZTlJXMDWfM5ckvnsHmb13Ajea7mW9ftpQ/37iO+flpnFCUybz8VFq7/Nx4xlx+fe1JMQN7JKUUnzt7PgAzc5IpzOz7jmUsfWTNzHC56IbT53DRsgLOHeSKzexUF9++bGl4MlqIsdK7W8Zq9e0d3Mcrcx+VIqBSag6wCngHKNBaV4PxAgDE/FeolLpJKbVZKbW5vr5+2D979axsLlxaQKcvwJfOL6HLH+SJ96v43tN7eGZnDR/+zZv89Ln9nLEgj8si9ucoKUjn02fOIzvFGW5T+sEHl1OQkcTe6jYuW1HIB5YWoLWxCOFss5vCabdRlJXE0zuqKavr4OPrZmFTipse3Mz9r5eT5LTx4w+tJD/d2LHvo2tmAnDJ8kLcDhu1bd6oU3I+sLSA0roO7nxyF8/truX60+eSnhRdbrBsOLGYB29YG7V8/fKVReSkurjprHlxn6P0JCdnlBhdOTab4sWvns0bt57HN9cv6VPaiOeCJQWsnJnFhUtidxONl1Wzsrn3k2sGNccixHiyMner5G51yiwqiJ25J/z2A0qpNOBx4Cta67bBngCjtb4PuA9gzZo1I+oDu/uaVdS3e5mVm8JL++q468VSGj0+rjt1Nq+VNlDZ3MV/b1jW53SaWy9exM3nLQhPdGUmO7nn46u4+eH3+ejJM1k8PZ2CDDft3QFOigjIs3NSeb2sgVSXnTvWL6HJ4+Pye17nxb11Ri04xcm3Ll3C/pp25pitbxlJTj6wbDr/3H4sqj574bLpfO9fe/nzO0c4sySPG08f2h7dJQXpbP32hUP6msjJnaF8zROfP01O+BEiDnuvFar7azvITnGGEz2LO9znnrg1d5RSTozA/het9d/Ny7VKqUKtdbVSqhAY2W5Og5DssjMr13jbfe26Wdz2950UZiZx+/olfD0Yor7dG17O3Wv8ffbsWDUrm9dvPS/8+ObzSmjt8kftd2K9xb/ixCJS3Q5S3Q7uvmYVn3lwM1efZGTqseq01582h/01bZy+oKe3vTgrmee+chZ5aa4+dfZEI4FdiPgiu2WqWrrYuLeWpUUZff7dJPzGYcoY8f3AXq31zyM+9RRwHfBD888nRzTCIbp8ZRH/9+4RPnf2fJKcdpKc9rhljsH4RIz699w8I7h/9OSePumzFuaz/c4P9FsuOGl2Ns9/9ew+12NtTiWEmFysIN7Y4eP2v++gyx/kjvVL+9xnsylc9rE/R3UkmfvpwCeBnUqpbea1b2IE9UeVUjcCR4CrRzbEoUl1O3jqi2eM6c/46JpZzMlN5cRe7W1SBxbi+GXtCvn9Z/ZytLmLh25YG/dsXLdj7I/aG3Zw11q/DsR7n37+cL/vZJCZ4uyzElEIcXyz+hy2H23hk6fM7nc3T7fTPuaZuyyZE0KIUWB1yyjFgIsqxyNzl+AuhBCjwJpQvWBJQcwN4iK5nTa8gdCYLmSS4C6EEKNgWnoSDpvis2fHX3NiSXLYqW/3suGeN2JuHjgaJvXGYUIIkSiWz8hk53cuGtQOn26njXcPNeGwKZYXZw54/3BI5i6EEKNksFs3Wwd2/PBDKzhtQd8zHUaDZO5CCDHOrjttNlecWDTgIe4jIcFdCCHG2cUnFA580whJWUYIIaYgCe5CCDEFSXAXQogpSIK7EEJMQRLchRBiCpLgLoQQU5AEdyGEmIIkuAshxBSktB7R8aWjMwil6oHDw/zyPKBhFIcz1mS8Y0vGO7ZkvGNrqOOdrbXOj/WJhAjuI6GU2qy1XjPR4xgsGe/YkvGOLRnv2BrN8UpZRgghpiAJ7kIIMQVNheB+30QPYIhkvGNLxju2ZLxja9TGO+lr7kIIIfqaCpm7EEKIXiS4CyHEFDSpg7tS6mKl1H6lVJlS6raJHk9vSqmZSqmXlFJ7lVK7lVJfNq9/RylVpZTaZv63fqLHalFKVSildprj2mxey1FKvaCUKjX/zJ7ocQIopRZFPIfblFJtSqmvJNLzq5T6g1KqTim1K+JazOdTGe42f593KKVWJ8h4f6KU2meO6R9KqSzz+hylVFfE8/zbBBlv3L9/pdTt5vO7Xyl1UYKM95GIsVYopbaZ10f2/GqtJ+V/gB04CMwDXMB2YOlEj6vXGAuB1ebH6cABYCnwHeDrEz2+OGOuAPJ6XfsxcJv58W3AjyZ6nHF+H2qA2Yn0/AJnAauBXQM9n8B64N+AAk4B3kmQ8X4AcJgf/yhivHMi70ug5zfm37/5b2874AbmmvHDPtHj7fX5nwH/NRrP72TO3NcCZVrrcq21D/grsGGCxxRFa12ttd5qftwO7AWKJ3ZUw7IBeMD8+AHgygkcSzznAwe11sNd6TwmtNavAk29Lsd7PjcAD2rD20CWUmrsz2OLEGu8WuvntdYB8+HbwNgd/DlEcZ7feDYAf9Vae7XWh4AyjDgybvobr1JKAR8BHh6NnzWZg3sxcDTicSUJHDiVUnOAVcA75qUvmm9z/5AoZQ6TBp5XSm1RSt1kXivQWleD8YIFTJuw0cV3DdH/KBL1+YX4z+dk+J2+AePdhWWuUup9pdQrSqkzJ2pQMcT6+0/05/dMoFZrXRpxbdjP72QO7irGtYTs61RKpQGPA1/RWrcBvwHmAycC1RhvxRLF6Vrr1cAlwBeUUmdN9IAGopRyAVcAj5mXEvn57U9C/04rpe4AAsBfzEvVwCyt9SrgFuD/lFIZEzW+CPH+/hP6+QU+RnSCMqLndzIH90pgZsTjGcCxCRpLXEopJ0Zg/4vW+u8AWutarXVQax0Cfsc4vzXsj9b6mPlnHfAPjLHVWuUB88+6iRthTJcAW7XWtZDYz68p3vOZsL/TSqnrgMuAa7VZEDbLG43mx1swatgLJ26Uhn7+/hP5+XUAHwQesa6N9PmdzMH9PaBEKTXXzNyuAZ6a4DFFMWto9wN7tdY/j7geWUe9CtjV+2snglIqVSmVbn2MMZG2C+N5vc687TrgyYkZYVxRGU+iPr8R4j2fTwH/YXbNnAK0WuWbiaSUuhi4FbhCa90ZcT1fKWU3P54HlADlEzPKHv38/T8FXKOUciul5mKM993xHl8cFwD7tNaV1oURP7/jOVM8BjPP6zE6UA4Cd0z0eGKM7wyMt307gG3mf+uBh4Cd5vWngMKJHqs53nkY3QTbgd3WcwrkAhuBUvPPnIkea8SYU4BGIDPiWsI8vxgvOtWAHyNzvDHe84lRNviV+fu8E1iTIOMtw6hVW7/DvzXv/ZD5e7Id2ApcniDjjfv3D9xhPr/7gUsSYbzm9T8Bn+1174ieX9l+QAghpqDJXJYRQggRhwR3IYSYgiS4CyHEFCTBXQghpiAJ7kIIMQVJcBdCiClIgrsQQkxB/x8rl/d7lBc87QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(score2);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.0245504   0.0344018  -0.00279386  0.02541287]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([1.8533], grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "S = ag_rb.env.reset()\n",
    "print(S)\n",
    "ag_rb.V(torch.tensor(S, dtype=torch.float))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([17.9258], grad_fn=<AddBackward0>)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ag_rb.V(torch.tensor([0, 0, 0, 0], dtype=torch.float))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}