EXPECTED_SARSA_LUNAR_LANDER.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "import gym\n",
    "from base_environment import BaseEnvironment\n",
    "from base_agent import BaseAgent\n",
    "from rl_glue import RLGlue\n",
    "import json\n",
    "from ast import literal_eval\n",
    "import time\n",
    "from IPython.display import clear_output\n",
    "#%matplotlib notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ReplayBuffer:\n",
    "    def __init__(self, size, batch_size, seed):\n",
    "        \"\"\"Initialize buffer paremeters\"\"\"\n",
    "        self.buffer = {\n",
    "            'states': [],\n",
    "            'actions': [],\n",
    "            'rewards': [],\n",
    "            'terminals': [],\n",
    "            'next_states': []\n",
    "        }\n",
    "        self.batch_size = batch_size\n",
    "        self.rand_generator = np.random.RandomState(seed)\n",
    "        self.max_size = size\n",
    "\n",
    "    def append(self, state, action, reward, terminal, next_state):\n",
    "        \"\"\"Append an experience\"\"\"\n",
    "        if len(self.buffer['states']) == self.max_size:\n",
    "            del self.buffer['states'][0]\n",
    "            del self.buffer['actions'][0]\n",
    "            del self.buffer['rewards'][0]\n",
    "            del self.buffer['terminals'][0]\n",
    "            del self.buffer['next_states'][0]\n",
    "        self.buffer['states'].append(state)\n",
    "        self.buffer['actions'].append(action)\n",
    "        self.buffer['rewards'].append(reward)\n",
    "        self.buffer['terminals'].append(terminal)\n",
    "        self.buffer['next_states'].append(next_state)\n",
    "        \n",
    "    def sample(self):\n",
    "        \"\"\"Returns a list of transition tuples including state, action, reward, terinal, and next_state\"\"\"\n",
    "        indices = self.rand_generator.choice(np.arange(len(self.buffer['states'])), size=self.batch_size)\n",
    "        return {\n",
    "            'states': [self.buffer['states'][i] for i in indices],\n",
    "            'actions': [self.buffer['actions'][i] for i in indices],\n",
    "            'rewards': [self.buffer['rewards'][i] for i in indices],\n",
    "            'terminals': [self.buffer['terminals'][i] for i in indices],\n",
    "            'next_states': [self.buffer['next_states'][i] for i in indices]\n",
    "        }\n",
    "    \n",
    "    def size(self):\n",
    "        return len(self.buffer['states'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ExpectedSarsaNetwork(tf.keras.Model):\n",
    "    def __init__(self, network_config):\n",
    "        super(ExpectedSarsaNetwork, self).__init__()\n",
    "        self.model_file = network_config['model_file']\n",
    "        self.dense1 = keras.layers.Dense(network_config['num_hidden_units'], input_dim=network_config['state_dim'], activation='relu')\n",
    "        self.dense2 = keras.layers.Dense(network_config['num_hidden_units'], activation='relu')\n",
    "        #self.dense3 = keras.layers.Dense(network_config['num_hidden_units'], activation='relu')\n",
    "        self.outputa = keras.layers.Dense(network_config['num_actions'], activation=None)\n",
    "    \n",
    "    def call(self, state):\n",
    "        x = self.dense1(state)\n",
    "        x = self.dense2(x)\n",
    "        #x = self.dense3(x)\n",
    "        x = self.outputa(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"HELPER SOFTMAX AND ARGMAX FUNCTIONS\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n",
    "def softmax_policy(action_values, tau=1.0):\n",
    "    \"\"\"Compute softmax probabilities from action values\"\"\"\n",
    "    # Compute the preferences by dividing the action-values by the temperature parameter tau\n",
    "    preferences = action_values / tau\n",
    "    # Compute the maximum preference across the actions\n",
    "    max_preference = np.max(action_values, axis=1) /tau\n",
    "    # Reshape max_preference array which has shape [Batch,] to [Batch, 1] (allow broadcast)\n",
    "    max_preference = max_preference.reshape((-1, 1))\n",
    "    # Compute the exponential of the preference - the max preference.\n",
    "    exp_preferences = np.exp(preferences - max_preference)\n",
    "    # Compute the sum over the numerator along the actions axis.\n",
    "    exp_preferences_sum = np.sum(exp_preferences, axis=1)    \n",
    "    # Reshape exp_preferences_sum array which has shape [Batch,] to [Batch, 1] (allow broadcast)\n",
    "    exp_preferences_sum = exp_preferences_sum.reshape((-1, 1))\n",
    "    # Compute the action probabilities\n",
    "    action_probabilities = exp_preferences / exp_preferences_sum\n",
    "    # Removes any singleton dimensions\n",
    "    action_probabilities = action_probabilities.squeeze()\n",
    "    # Replace any unnoticed NaNs with 0\n",
    "    where_are_NaNs = np.isnan(action_probabilities)\n",
    "    action_probabilities[where_are_NaNs] = 0\n",
    "    # Returning the action probabilities\n",
    "    return action_probabilities\n",
    "\n",
    "def argmax(action_values, random_generator):\n",
    "    \"\"\"argmax with random tie breaking\"\"\"\n",
    "    return np.random.choice(np.flatnonzero(action_values == action_values.max()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ExpectedSarsaAgent(BaseAgent):\n",
    "    \"\"\"Implements the agent for an RL-Glue environment\"\"\" \n",
    "    def __init__(self):\n",
    "        self.name = \"expected_sarsa_agent\"\n",
    "\n",
    "    def agent_init(self, agent_info={}):\n",
    "        \"\"\"Setup for the agent called when the experiment first starts.\"\"\"\n",
    "        # Number of actions\n",
    "        self.num_actions = agent_info['network_config']['num_actions']\n",
    "        # Number of experience replay steps\n",
    "        self.num_replay = agent_info['num_replay_updates_per_step']\n",
    "        # Discount factor\n",
    "        self.discount = agent_info['gamma']\n",
    "        # Tau the temperature parameter used in the softmax policy\n",
    "        self.tau = agent_info['tau']\n",
    "        # Discount factor\n",
    "        self.step_size = agent_info['optimizer_config']['step_size']\n",
    "        # A random  generator used with a knowen seed\n",
    "        self.rand_generator = np.random.RandomState(agent_info.get(\"seed\"))\n",
    "        # Replay Buffer used instead of a learned model\n",
    "        self.replay_buffer = ReplayBuffer(agent_info['replay_buffer_size'], \n",
    "                                          agent_info['batch_size'], agent_info.get(\"seed\"))\n",
    "        \n",
    "        # Defining the using_softmax_policy parameter\n",
    "        self.using_softmax_policy = agent_info['using_softmax_policy']\n",
    "        # Check if using the e greedy action selection\n",
    "        if self.using_softmax_policy:\n",
    "            # Tau the temperature parameter used in softmax action selection\n",
    "            self.tau = agent_info['tau']\n",
    "        else:\n",
    "            # Epsilon for E-Greedy action selection\n",
    "            self.e_greedy_m = agent_info['e_greedy_m']\n",
    "            self.e_greedy_min = agent_info['e_greedy_min']\n",
    "            self.e_greedy_epsilon = ((self.e_greedy_min - 1) / self.e_greedy_m) * 1 + 1\n",
    "            \n",
    "        \n",
    "\n",
    "        # Initialize the evaluation network\n",
    "        self.target_evaluation_network = ExpectedSarsaNetwork(agent_info['network_config'])\n",
    "\n",
    "        # Initialize the next network \n",
    "        self.network = ExpectedSarsaNetwork(agent_info['network_config'])\n",
    "\n",
    "        # compile the evaluation network\n",
    "        self.target_evaluation_network.compile(optimizer=keras.optimizers.Adam(\n",
    "            lr=self.step_size), loss=\"mse\",metrics=[\"mse\"])\n",
    "        \n",
    "        # Call the subclassed model once to create the weights.\n",
    "        self.target_evaluation_network(tf.ones((agent_info['batch_size'],agent_info['network_config']['state_dim'] )))\n",
    "\n",
    "        # compile the next nework\n",
    "        self.network.compile(optimizer=keras.optimizers.Adam(\n",
    "            lr=self.step_size), loss=\"mse\", metrics=[\"mse\"])\n",
    "        \n",
    "        # Initialising paremeters\n",
    "        self.last_state = None\n",
    "        self.last_action = None\n",
    "        \n",
    "        self.dict_info = {\n",
    "            'run_info': {\n",
    "                'softmax_or_e_greedy': '',\n",
    "                'step-size': -1\n",
    "            },\n",
    "            'loss': [],\n",
    "            'metric': [],\n",
    "            'episode_steps': [],\n",
    "            'rewards': [],\n",
    "            'model': ''\n",
    "        }\n",
    "        \n",
    "    def choose_e_greedy_action(self, observation):\n",
    "        \"\"\"Used to choose an action\"\"\"\n",
    "        # Get the action values from the neural network\n",
    "        observation_tensor = tf.Variable(observation, dtype=tf.float32)\n",
    "        # Convert action values type from tensors to numpy array\n",
    "        action_values_tensor = self.network.call(observation_tensor)\n",
    "        action_values_proto_tensor = tf.make_tensor_proto(action_values_tensor)\n",
    "        action_values = tf.make_ndarray(action_values_proto_tensor)\n",
    "        # Check the value of epsilon\n",
    "        if np.random.random() < self.e_greedy_epsilon:\n",
    "            return  np.random.choice([i for i in range(len(action_values))])\n",
    "        else:\n",
    "            return argmax(action_values, self.rand_generator)\n",
    "    \n",
    "    def choose_softmax_action(self, observation):\n",
    "        \"\"\"Used to choose an action\"\"\"\n",
    "        # Create a tensor variable\n",
    "        observation_tensor = tf.Variable(observation, dtype=tf.float32)\n",
    "        # Get the action values from the neural network\n",
    "        action_values_tensor = self.network.call(observation_tensor)\n",
    "        # Convert action values type from tensors to numpy array\n",
    "        action_values_proto_tensor = tf.make_tensor_proto(action_values_tensor)\n",
    "        action_values = tf.make_ndarray(action_values_proto_tensor)\n",
    "        # Get the actions probabilities from the softmax using action values\n",
    "        probs_batch = softmax_policy(action_values, self.tau)\n",
    "        # Choosing an action using the probability distribution generated by the softmax policy\n",
    "        action = self.rand_generator.choice(self.num_actions, p=probs_batch.squeeze())\n",
    "        # returning the chosen action\n",
    "        return action\n",
    "        \n",
    "    def agent_start(self, observation):\n",
    "        \"\"\"The first method called when the experiment starts\"\"\"\n",
    "        # Setting the last state \n",
    "        self.last_state = np.array([observation])\n",
    "        \n",
    "#         if len(self.dict_info['episode_steps']) == 200:\n",
    "#             self.tau = 0.000001\n",
    "        \n",
    "        # check for softmax or e_greedy\n",
    "        if self.using_softmax_policy:\n",
    "            # Setting the last action using the softmax policy method\n",
    "            self.last_action = self.choose_softmax_action(self.last_state)\n",
    "        else:\n",
    "            # Setting the last action using the epsilon greedy policy method\n",
    "            self.last_action = self.choose_e_greedy_action(self.last_state)\n",
    "        # Add an element to dict_info['rewards'] list\n",
    "        self.dict_info['rewards'].append(0)\n",
    "        # Add an element to dict_info['episode_steps'] list\n",
    "        self.dict_info['episode_steps'].append(0)\n",
    "        # Returing the last action\n",
    "        return self.last_action\n",
    "\n",
    "    \n",
    "    def agent_step(self, reward, observation):\n",
    "        \"\"\"A step taken by the agent\"\"\"\n",
    "        # convert the observation to a numpy array\n",
    "        state = np.array([observation])\n",
    "        # check for softmax or e_greedy\n",
    "        if self.using_softmax_policy:\n",
    "            # selecting the action using the softmax policy \n",
    "            action = self.choose_softmax_action(state)\n",
    "        else:\n",
    "            # Applying Annealling e_greedy_epsilon equation (len(self.dict_info['episode_steps']) == number of episodes)\n",
    "            update = ((self.e_greedy_min - 1) / self.e_greedy_m) * len(self.dict_info['episode_steps']) + 1\n",
    "            if update > self.e_greedy_min:\n",
    "                self.e_greedy_epsilon = update\n",
    "            else:\n",
    "                self.e_greedy_epsilon = self.e_greedy_min\n",
    "            # selecting the action using the epsilon greedy policy\n",
    "            action = self.choose_e_greedy_action(state)\n",
    "        # Append new experience to replay buffer\n",
    "        self.replay_buffer.append(self.last_state, self.last_action, reward, 0, state)\n",
    "        # Perform replay steps\n",
    "        if self.replay_buffer.size() > self.replay_buffer.batch_size:\n",
    "            # Copy the network weights to the target_evaluation_network\n",
    "            self.target_evaluation_network.set_weights(self.network.get_weights())\n",
    "            # For each replay step\n",
    "            for _ in range(self.num_replay):\n",
    "                # Get sample experiences from the replay buffer\n",
    "                experiences = self.replay_buffer.sample()\n",
    "                # Converting experiences to numpy arrays\n",
    "                states = np.array(experiences['states']).squeeze()\n",
    "                actions = np.array(experiences['actions']).squeeze()\n",
    "                rewards = np.array(experiences['rewards']).squeeze()\n",
    "                terminals = np.array(experiences['terminals']).squeeze()\n",
    "                next_states = np.array(experiences['next_states']).squeeze()\n",
    "                # Reshaping rewards and terminals to [batch-size, 1], to broadcast operations\n",
    "                rewards = rewards.reshape((-1, 1))\n",
    "                terminals = terminals.reshape((-1, 1))\n",
    "                # Compute action values at current states using network\n",
    "                td_target = self.network(states).numpy()\n",
    "                # Compute action values at next states using target_evaluation_network\n",
    "                q_target_matrix = self.target_evaluation_network(next_states).numpy()\n",
    "                # check for softmax or e_greedy\n",
    "                if self.using_softmax_policy:\n",
    "                    # Compute policy probabilities at next state by passing the action-values to softmax_policy()    \n",
    "                    probabilities_matrix = softmax_policy(q_target_matrix, self.tau)\n",
    "                else:\n",
    "                    # Compute actions probabilities in the case of EPSILON GREEDY ACTION SELECTION\n",
    "                    q_max = np.max(q_target_matrix, axis=1).reshape((-1, 1))\n",
    "                    probabilities_matrix = np.ones_like(q_target_matrix) * (self.e_greedy_epsilon / self.num_actions)\n",
    "                    for i in range(probabilities_matrix.shape[0]):\n",
    "                        probabilities_matrix[i] += (q_target_matrix[i] == q_max[i]) * ((1 - self.e_greedy_epsilon) / np.sum(q_target_matrix[i] == q_max[i]))\n",
    "                # Compute the estimate of the next state value, v_target_vector\n",
    "                # NOTE: this line is the only difference between Expected Sarsa and Q-learning\n",
    "                # Changing this line from the sum of (next q-values * their probabilities) \n",
    "                # to the maximum q-value will just make it Q-Learning\n",
    "                v_target_vector = np.sum(q_target_matrix * probabilities_matrix, axis=1).reshape((-1, 1)) * (1 - terminals) \n",
    "                # Q-learning\n",
    "                #v_target_vector = np.max(q_target_matrix, axis=1) * (1 - terminals)\n",
    "                # Compute Expected Sarsa target \n",
    "                targets = rewards + self.discount * v_target_vector\n",
    "                for i in range(terminals.shape[0]):\n",
    "                    td_target[i, actions[i]] = targets[i]\n",
    "                # Training the network using the batch of (states, TD_target)\n",
    "                r = self.network.train_on_batch(states, td_target)\n",
    "                # Saving loss and metric to dict info\n",
    "                self.dict_info['loss'].append(r[0])\n",
    "                self.dict_info['metric'].append(r[1])\n",
    "                \n",
    "        # Update the last state\n",
    "        self.last_state = state\n",
    "        # Update the last action\n",
    "        self.last_action = action\n",
    "        # Adding the last step resulting reward to dict_info['rewards']\n",
    "        self.dict_info['rewards'][-1] += reward\n",
    "        # Adding an episode step\n",
    "        self.dict_info['episode_steps'][-1] += 1\n",
    "        # Return the selected action\n",
    "        return action\n",
    "    \n",
    "    def agent_end(self, reward):\n",
    "        \"\"\"The final step taken by the agent\"\"\"\n",
    "        # Setting the state as an array of zeros\n",
    "        state = np.zeros_like(self.last_state)\n",
    "        # Check for softmax or e_greedy\n",
    "        if not self.using_softmax_policy:\n",
    "            # Applying Annealling e_greedy_epsilon equation (len(self.dict_info['episode_steps']) == number of episodes)\n",
    "            update = ((self.e_greedy_min - 1) / self.e_greedy_m) * len(self.dict_info['episode_steps']) + 1\n",
    "            if update > self.e_greedy_min:\n",
    "                self.e_greedy_epsilon = update\n",
    "            else:\n",
    "                self.e_greedy_epsilon = self.e_greedy_min\n",
    "        # Append new experience to replay buffer\n",
    "        self.replay_buffer.append(self.last_state, self.last_action, reward, 0, state)\n",
    "        \n",
    "        # Perform replay steps:\n",
    "        if self.replay_buffer.size() > self.replay_buffer.batch_size:\n",
    "            # Copy the neural network ???\n",
    "            self.target_evaluation_network.set_weights(self.network.get_weights())\n",
    "            # For each replay step\n",
    "            for _ in range(self.num_replay):\n",
    "                # Get sample experiences from the replay buffer\n",
    "                experiences = self.replay_buffer.sample()\n",
    "                # Converting experiences to numpy arrays\n",
    "                states = np.array(experiences['states']).squeeze()\n",
    "                actions = np.array(experiences['actions']).squeeze()\n",
    "                rewards = np.array(experiences['rewards']).squeeze()\n",
    "                terminals = np.array(experiences['terminals']).squeeze()\n",
    "                next_states = np.array(experiences['next_states']).squeeze()\n",
    "                # Reshaping rewards and terminals to [batch-size, 1], to broadcast operations\n",
    "                rewards = rewards.reshape((-1, 1))\n",
    "                terminals = terminals.reshape((-1, 1))\n",
    "                # Compute action values at current states using network\n",
    "                td_target = self.network(states).numpy()\n",
    "                # Compute action values at next states using target_evaluation_network\n",
    "                q_target_matrix = self.target_evaluation_network(next_states).numpy()\n",
    "                # check for softmax or e_greedy\n",
    "                if self.using_softmax_policy:\n",
    "                    # Compute policy probabilities at next state by passing the action-values to softmax_policy()   \n",
    "                    probabilities_matrix = softmax_policy(q_target_matrix, self.tau)\n",
    "                else:\n",
    "                    # Compute actions probabilities in the case of EPSILON GREEDY ACTION SELECTION\n",
    "                    q_max = np.max(q_target_matrix, axis=1).reshape((-1, 1))\n",
    "                    probabilities_matrix = np.ones_like(q_target_matrix) * (self.e_greedy_epsilon / self.num_actions)\n",
    "                    for i in range(probabilities_matrix.shape[0]):\n",
    "                        probabilities_matrix[i] += (q_target_matrix[i] == q_max[i]) * ((1 - self.e_greedy_epsilon) / np.sum(q_target_matrix[i] == q_max[i]))\n",
    "                # Compute the estimate of the next state value, v_target_vector\n",
    "                # NOTE: this line is the only difference between Expected Sarsa and Q-learning\n",
    "                # Changing this line from the sum of (next q-values * their probabilities) \n",
    "                # to the maximum q-value will just make it Q-Learning\n",
    "                v_target_vector = np.sum(q_target_matrix * probabilities_matrix, axis=1).reshape((-1, 1)) * (1 - terminals) \n",
    "                # Q-learning\n",
    "                #v_target_vector = np.max(q_target_matrix, axis=1) * (1 - terminals)\n",
    "                # Compute Expected Sarsa target \n",
    "                targets = rewards + self.discount * v_target_vector\n",
    "                for i in range(terminals.shape[0]):\n",
    "                    td_target[i, actions[i]] = targets[i]\n",
    "                # Training the network using the batch of (states, TD_target)\n",
    "                r = self.network.train_on_batch(states, td_target)\n",
    "                # Saving loss and metric to dict info\n",
    "                self.dict_info['loss'].append(r[0])\n",
    "                self.dict_info['metric'].append(r[1])\n",
    "                \n",
    "        \n",
    "        \n",
    "        # Adding the last step resulting reward to dict_info['rewards']\n",
    "        self.dict_info['rewards'][-1] += reward\n",
    "        # Adding an episode step\n",
    "        self.dict_info['episode_steps'][-1] += 1\n",
    "        \n",
    "    def agent_message(self, message):\n",
    "        \"\"\"A function used to pass information from the agent to the experiment\"\"\"\n",
    "        if message == \"get_dict_info\":\n",
    "            return self.dict_info\n",
    "        elif message == \"get_model\":\n",
    "            return self.network\n",
    "        elif message == \"get_episode_reward\":\n",
    "            return self.dict_info['rewards'][-1]\n",
    "        \n",
    "        else:\n",
    "            raise Exception(\"Unrecognized Message!\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LunarLanderEnvironment(BaseEnvironment):\n",
    "    def env_init(self, env_info={}):\n",
    "        \"\"\"Setup for the environment called when the experiment first starts\"\"\"\n",
    "        # Initialize env to LunarLander-v2\n",
    "        self.env = gym.make(env_info['environment_name'])\n",
    "        # Set a random seed, \n",
    "        # NOTE: this is important if we want to do tests for multiple algorithms \n",
    "        # or multiple algorithm parameters. \n",
    "        self.env.seed(0)\n",
    "\n",
    "    def env_start(self):\n",
    "        \"\"\"The first method called when the experiment starts\"\"\"\n",
    "        # Get Initial State\n",
    "        observation = self.env.reset()\n",
    "        # Return the initial state from the environment\n",
    "        return observation\n",
    "        \n",
    "    def env_step(self, action):\n",
    "        \"\"\"A step taken by the environment\"\"\"\n",
    "        #self.env.render()\n",
    "        # Get state, reward and terminal from the environment\n",
    "        state, reward, terminal, _ = self.env.step(action)\n",
    "        # Return the reward, state and terminal to the RL Glue\n",
    "        return (reward, state, terminal)\n",
    "    def env_cleanup(self):\n",
    "        \"\"\"Cleanup done after the environment ends\"\"\"\n",
    "        #self.env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"HELPER LOAD, PARSE and PLOT FUNCTIONS\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n",
    "def parse_loaded_data(load):\n",
    "    load['loss'] = literal_eval(load['loss'])\n",
    "    load['metric'] = literal_eval(load['metric'])\n",
    "    load['rewards'] = literal_eval(load['rewards'])\n",
    "    load['loss'] = [float(i) for i in load['loss']]\n",
    "    load['metric'] = [float(i) for i in load['metric']]\n",
    "    load['rewards'] = [float(i) for i in load['rewards']]\n",
    "    return load\n",
    "\n",
    "def load_data(loaded, data_name):\n",
    "    if data_name == 'batch_size_and_replay_steps':\n",
    "        for i in range(len(agent_info['batch_sizes'])): \n",
    "            for j in range(len(agent_info['replays_steps'])):\n",
    "                with open('.\\\\results\\\\batch_size_and_replay_steps\\\\data_softmax_b' + str(i) + '_ns' + str(j) + '.json') as file: \n",
    "                    loaded['batch_size_and_replay_steps'].append(parse_loaded_data(json.load(file)))\n",
    "    elif data_name == 'e_greedy_and_softmax':\n",
    "        for i in range(len(experiment_info['tau_values'])): \n",
    "            for j in range(len(experiment_info['step_sizes'])):\n",
    "                with open('.\\\\results\\\\e_greedy_and_softmax\\\\data_softmax_t' + str(i) + '_s' + str(j) + '.json') as file: \n",
    "                    loaded['softmax_tau_step_size'].append(parse_loaded_data(json.load(file)))\n",
    "        for i in range(len(experiment_info['e_greedy_m_values'])): \n",
    "            for j in range(len(experiment_info['step_sizes'])):\n",
    "                with open('.\\\\results\\\\e_greedy_and_softmax\\\\data_e_greedy_e' + str(i) + '_s' + str(j) + '.json') as file: \n",
    "                    loaded['e_greedy_epsilon_step_size'].append(parse_loaded_data(json.load(file)))\n",
    "    elif data_name == 'best_model_data':\n",
    "        with open('.\\\\results\\\\best_model\\\\data_softmax.json') as file: \n",
    "            loaded['best_model_data'] = parse_loaded_data(json.load(file))\n",
    "    \n",
    "def reconstruct_model(path):\n",
    "    # Initialize the network\n",
    "    network = ExpectedSarsaNetwork(agent_info['network_config'])\n",
    "    # compile the network\n",
    "    network.compile(optimizer=keras.optimizers.Adam(\n",
    "        lr=agent_info['optimizer_config']['step_size']), loss=\"mse\",metrics=[\"mse\"])\n",
    "    # Call the subclassed model once to create the weights.\n",
    "    network.call(tf.ones((agent_info['batch_size'],agent_info['network_config']['state_dim'] )))\n",
    "    # Set the weights of the neural network\n",
    "    network.load_weights(path)\n",
    "    # Return the network\n",
    "    return network\n",
    "\n",
    "def compute_average(data):\n",
    "    for i in range(len(data)):\n",
    "        data[i]['average_loss'] = np.sum(np.array(data[i]['loss'])) / len(data[i]['loss'])\n",
    "        data[i]['average_metric'] = np.sum(np.array(data[i]['metric'])) / len(data[i]['metric'])\n",
    "        data[i]['average_reward'] = np.sum(np.array(data[i]['rewards'])) / len(data[i]['rewards'])\n",
    "        \n",
    "\n",
    "def moving_average(a, n=10) :\n",
    "    \"\"\"Moving Average or (Sliding window) used to make the plot smoother\"\"\"\n",
    "    ret = np.cumsum(a, dtype=float)\n",
    "    ret[n:] = ret[n:] - ret[:-n]\n",
    "    return ret[n - 1:] / n\n",
    "\n",
    "    \n",
    "def plot(data, run_info, data_name, only_smoothed, smoothing_step, title, x_label, y_label, fig_size=(13,4), ylim=False, ylim_val=[0, 10], clear=False):\n",
    "    if clear:\n",
    "        plt.clf\n",
    "        clear_output(wait=True)\n",
    "    if data_name == 'batch_size_and_replay_steps':\n",
    "        print('batch size :', run_info['batch'], ', replay steps :', run_info['num_steps'])\n",
    "        print()\n",
    "    elif data_name == 'softmax_tau_step_size':\n",
    "        print('tau :', run_info['tau'], ', step size :', run_info['step_size'])\n",
    "        print()\n",
    "    elif data_name == 'e_greedy_epsilon_step_size':\n",
    "        print('e greedy M :', run_info['epsilon'], ', step size :', run_info['step_size'])\n",
    "        print()\n",
    "    if only_smoothed == 0 or only_smoothed == 2:\n",
    "        plt.figure(figsize=fig_size)\n",
    "        plt.title(title, fontsize = 15)\n",
    "        plt.xlabel(x_label, fontsize = 14)\n",
    "        plt.ylabel(y_label, rotation=0, labelpad=40, fontsize = 14)\n",
    "        if ylim == True:\n",
    "            plt.ylim(ylim_val)\n",
    "        plt.plot(data)\n",
    "        plt.show() \n",
    "    elif only_smoothed == 1 or only_smoothed == 2:\n",
    "        plt.figure(figsize=(12,4))\n",
    "        plt.title(title, fontsize = 15)\n",
    "        plt.xlabel(x_label, fontsize = 14)\n",
    "        plt.ylabel(y_label, rotation=0, labelpad=40, fontsize = 14)\n",
    "        smoothed_data = moving_average(data, smoothing_step)\n",
    "        plt.plot(smoothed_data)\n",
    "        plt.show()  \n",
    "        \n",
    "    \n",
    "def plot_same_type(data_type, data_name, only_smoothed):\n",
    "    print('Number of test instances :', len(loaded[data_name]))\n",
    "    print()\n",
    "    if data_type == 'loss':\n",
    "        for i in range(len(loaded[data_name])):\n",
    "            game = loaded[data_name][i]\n",
    "            plot(game['loss'], game['run_info'], data_name, only_smoothed, 50000, 'Loss of '+ data_name, 'steps', 'Loss')\n",
    "    elif data_type == 'rewards':\n",
    "        for i in range(len(loaded[data_name])):\n",
    "            game = loaded[data_name][i]\n",
    "            plot(game['rewards'], game['run_info'], data_name, only_smoothed, 30, 'Reward of '+ data_name, 'Episodes', 'rewards')\n",
    "    elif data_type == 'episode_steps':\n",
    "        for i in range(len(loaded[data_name])):\n",
    "            game = loaded[data_name][i]\n",
    "            plot(game['episode_steps'], game['run_info'], data_name, only_smoothed, 100, 'Number of steps in each episode of '+ data_name, 'Episodes', 'num_steps')\n",
    "    elif data_type == 'metric':\n",
    "        for i in range(len(loaded[data_name])):\n",
    "            game = loaded[data_name][i]\n",
    "            plot(game['metric'], game['run_info'], data_name, only_smoothed, 50000, 'Learning Curve', 'steps', 'metric')\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"Testing e_greedy and softmax\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n",
    "def run_experiment(environment, agent, runs_info, environment_parameters, agent_parameters, experiment_parameters):\n",
    "    # Instantiate an RLGlue Object\n",
    "    rl_glue = RLGlue(environment, agent)\n",
    "    \n",
    "    # Setting the run random seeds to get consistent results each time we do an experement\n",
    "    agent_parameters[\"seed\"] = 1\n",
    "    agent_parameters[\"network_config\"][\"seed\"] = 1\n",
    "    environment_parameters[\"seed\"] = 1\n",
    "    \n",
    "    ######################## E_GREEDY ########################\n",
    "    agent_parameters['using_softmax_policy'] = False\n",
    "    print('E-greedy action selection')\n",
    "    for i, e_greedy_m in enumerate(experiment_parameters['e_greedy_m_values']): \n",
    "        for j, step_size in enumerate(experiment_parameters['step_sizes']):\n",
    "            agent_parameters['e_greedy_m'] = e_greedy_m\n",
    "            agent_parameters['optimizer_config']['step_size'] = step_size\n",
    "            # Initialize the RLGlue Object\n",
    "            rl_glue.rl_init(agent_parameters, environment_parameters)\n",
    "            # For each episode\n",
    "            for episode in tqdm(range(experiment_parameters[\"num_episodes\"])):\n",
    "                # Call an RLGlue episode\n",
    "                rl_glue.rl_episode(experiment_parameters[\"timeout\"])\n",
    "\n",
    "            # Getting dict_info using agent_message function\n",
    "            runs_info.append(rl_glue.rl_agent_message(\"get_dict_info\"))\n",
    "            # Appending to runs_info the info of this run\n",
    "            runs_info[-1]['run_info']['softmax_or_e_greedy'] = 'e_greedy'\n",
    "            runs_info[-1]['run_info']['epsilon'] = str(e_greedy_m)\n",
    "            runs_info[-1]['run_info']['step_size'] = str(step_size)\n",
    "            runs_info[-1]['rewards'] = str(runs_info[-1]['rewards'])\n",
    "            runs_info[-1]['loss'] = str(runs_info[-1]['loss'])\n",
    "            runs_info[-1]['metric'] = str(runs_info[-1]['metric'])\n",
    "            # Save runs_info in a text file\n",
    "            with open('.\\\\tests\\\\test1\\\\data_e_greedy_e' + str(i) + '_s' + str(j) + '.json', 'w') as file:\n",
    "                json.dump(runs_info[-1], file)\n",
    "            # Getting the model using agent_message\n",
    "            model = rl_glue.rl_agent_message(\"get_model\") \n",
    "            # Saving the weights of the model in a file\n",
    "            model.save_weights('.\\\\tests\\\\test1\\\\model_e_greedy_e' + str(i) + '_s' + str(j) + '.h5')\n",
    "            # appending the model to runs_info\n",
    "            runs_info[-1]['model'] = model\n",
    "\n",
    "    ######################## SOFTMAX #########################\n",
    "    agent_parameters['using_softmax_policy'] = True                   \n",
    "    print('Softmax action selection')\n",
    "    for i, tau in enumerate(experiment_parameters['tau_values']): \n",
    "        for j, step_size in enumerate(experiment_parameters['step_sizes']):\n",
    "            agent_parameters['tau'] = tau\n",
    "            agent_parameters['optimizer_config']['step_size'] = step_size\n",
    "            # Initializing the RLGlue Object\n",
    "            rl_glue.rl_init(agent_parameters, environment_parameters)\n",
    "            # For each episode\n",
    "            for episode in tqdm(range(experiment_parameters[\"num_episodes\"])):\n",
    "                # Calling an RLGlue episode\n",
    "                rl_glue.rl_episode(experiment_parameters[\"timeout\"])\n",
    "\n",
    "            # Getting dict_info using agent_message function\n",
    "            runs_info.append(rl_glue.rl_agent_message(\"get_dict_info\"))\n",
    "            # Appending to runs_info the info of this run\n",
    "            runs_info[-1]['run_info']['softmax_or_e_greedy'] = 'softmax'\n",
    "            runs_info[-1]['run_info']['tau'] = str(tau)\n",
    "            runs_info[-1]['run_info']['step_size'] = str(step_size)\n",
    "            runs_info[-1]['rewards'] = str(runs_info[-1]['rewards'])\n",
    "            runs_info[-1]['loss'] = str(runs_info[-1]['loss'])\n",
    "            runs_info[-1]['metric'] = str(runs_info[-1]['metric'])\n",
    "\n",
    "            # Save runs_info in a text file\n",
    "            with open('.\\\\tests\\\\test2\\\\data_softmax_t' + str(i) + '_s' + str(j) + '.json', 'w') as file:\n",
    "                json.dump(runs_info[-1], file)\n",
    "            # Getting the model using agent_message\n",
    "            model = rl_glue.rl_agent_message(\"get_model\") \n",
    "            # Saving the weights of the model in a file\n",
    "            model.save_weights('.\\\\tests\\\\test2\\\\model_softmax_t' + str(i) + '_s' + str(j) + '.h5')\n",
    "            # appending the model to runs_info\n",
    "            runs_info[-1]['model'] = model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"Testing Batch Size and Replay Steps\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n",
    "def run_experiment(environment, agent, runs_info, environment_parameters, agent_parameters, experiment_parameters):\n",
    "    # Instantiate an RLGlue Object\n",
    "    rl_glue = RLGlue(environment, agent)\n",
    "    # For each run\n",
    "    agent_parameters[\"seed\"] = 1\n",
    "    agent_parameters[\"network_config\"][\"seed\"] = 1\n",
    "    environment_parameters[\"seed\"] = 1\n",
    "    for i, batch in enumerate(agent_parameters['batch_sizes']):\n",
    "        for j, num_steps in enumerate(agent_parameters['replays_steps']):\n",
    "            agent_parameters['batch_size'] = batch\n",
    "            agent_parameters['num_replay_updates_per_step'] = num_steps\n",
    "            # Initialize the RLGlue Object\n",
    "            rl_glue.rl_init(agent_parameters, environment_parameters)\n",
    "            # For each episode\n",
    "            for episode in tqdm(range(experiment_parameters[\"num_episodes\"])):\n",
    "                # Call an RLGlue episode\n",
    "                rl_glue.rl_episode(experiment_parameters[\"timeout\"])\n",
    "                \n",
    "            runs_info.append(rl_glue.rl_agent_message(\"get_dict_info\"))\n",
    "            runs_info[-1]['run_info']['softmax_or_e_greedy'] = 'softmax'\n",
    "            runs_info[-1]['run_info']['batch'] = str(batch)\n",
    "            runs_info[-1]['run_info']['num_steps'] = str(num_steps)\n",
    "            runs_info[-1]['rewards'] = str(runs_info[-1]['rewards'])\n",
    "            runs_info[-1]['loss'] = str(runs_info[-1]['loss'])\n",
    "            runs_info[-1]['metric'] = str(runs_info[-1]['metric'])\n",
    "\n",
    "            # Save dict_info in a text file\n",
    "            with open('.\\\\tests\\\\test3\\\\data_softmax_b' + str(i) + '_ns' + str(j) + '.json', 'w') as file:\n",
    "                json.dump(runs_info[-1], file)\n",
    "\n",
    "            # Getting the model using agent_message\n",
    "            model = rl_glue.rl_agent_message(\"get_model\") \n",
    "            # Saving the weights of the model in a file\n",
    "            model.save_weights('.\\\\tests\\\\test3\\\\model_softmax_b' + str(i) + '_ns' + str(j) + '.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"Normal Run\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\"\n",
    "def run_experiment(environment, agent, runs_info, environment_parameters, agent_parameters, experiment_parameters):\n",
    "    # Instantiate an RLGlue Object\n",
    "    rl_glue = RLGlue(environment, agent)\n",
    "    # For each run\n",
    "    agent_parameters[\"seed\"] = 1\n",
    "    agent_parameters[\"network_config\"][\"seed\"] = 1\n",
    "    environment_parameters[\"seed\"] = 1\n",
    "    # Initialize the RLGlue Object\n",
    "    rl_glue.rl_init(agent_parameters, environment_parameters)\n",
    "    # For each episode\n",
    "    plot_reward = []\n",
    "    for episode in tqdm(range(experiment_parameters[\"num_episodes\"])):\n",
    "        # Call an RLGlue episode\n",
    "        rl_glue.rl_episode(experiment_parameters[\"timeout\"])\n",
    "        plot_reward.append(rl_glue.rl_agent_message(\"get_episode_reward\"))\n",
    "        plot(plot_reward, [], '', 0, 30, 'Reward', 'Episodes', 'Learning Curve', clear=True)\n",
    "        \n",
    "    rl_glue.rl_cleanup()\n",
    "    runs_info.append(rl_glue.rl_agent_message(\"get_dict_info\"))\n",
    "    runs_info[-1]['run_info']['softmax_or_e_greedy'] = 'softmax'\n",
    "    runs_info[-1]['run_info']['batch'] = str(8)\n",
    "    runs_info[-1]['run_info']['num_steps'] = str(8)\n",
    "    runs_info[-1]['run_info']['tau'] = str(0.01)\n",
    "    runs_info[-1]['run_info']['step_size'] = str(0.0005)\n",
    "    runs_info[-1]['rewards'] = str(runs_info[-1]['rewards'])\n",
    "    runs_info[-1]['loss'] = str(runs_info[-1]['loss'])\n",
    "    runs_info[-1]['metric'] = str(runs_info[-1]['metric'])\n",
    "   \n",
    "    # Save dict_info in a text file\n",
    "    with open('.\\\\results\\\\data_softmax.json', 'w') as file:\n",
    "        json.dump(runs_info[-1], file)\n",
    "\n",
    "    # Getting the model using agent_message\n",
    "    model = rl_glue.rl_agent_message(\"get_model\") \n",
    "    # Saving the weights of the model in a file\n",
    "    model.save_weights('.\\\\results\\\\model_softmax.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "agent_info = {\n",
    "    'network_config': {\n",
    "        'state_dim': 8,\n",
    "        'num_hidden_units': 128,\n",
    "        'num_actions': 4,\n",
    "        'model_file': 'network_model.h5'\n",
    "    },\n",
    "    'optimizer_config': {\n",
    "        'step_size': 0.0005\n",
    "    },\n",
    "    'replay_buffer_size': 50000,\n",
    "    'batch_size': 8,\n",
    "    'num_replay_updates_per_step': 8,\n",
    "    'gamma': 0.99,\n",
    "    'tau': 0.01,\n",
    "    'e_greedy_m': 20,\n",
    "    'e_greedy_min': 0.01,\n",
    "    'using_softmax_policy': True,\n",
    "    # For parameter tuning\n",
    "    'batch_sizes': [8,16,32,64,128],\n",
    "    'replays_steps': [2,4,8,16]\n",
    "}\n",
    "experiment_info = {\n",
    "    'num_episodes' : 3,\n",
    "    'timeout' : 1000,\n",
    "    # For parameter tuning\n",
    "    'tau_values': np.array([0.001, 0.01, 0.1, 1.0]),\n",
    "    'e_greedy_m_values': np.array([20, 50, 100]),\n",
    "    'step_sizes': np.array([0.0001, 0.0005, 0.001, 0.01, 0.1])\n",
    "}\n",
    "env_info = {\n",
    "    'environment_name': 'LunarLander-v2'\n",
    "}\n",
    "lunar_lander = LunarLanderEnvironment\n",
    "expected_sarsa_agent = ExpectedSarsaAgent\n",
    "runs_info = []\n",
    "# run experiment\n",
    "\n",
    "run_experiment(lunar_lander, expected_sarsa_agent, runs_info, env_info, agent_info, experiment_info)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# RECONSTRUCT RUN_INFO\n",
    "loaded = {\n",
    "    'batch_size_and_replay_steps': [],\n",
    "    'softmax_tau_step_size': [],\n",
    "    'e_greedy_epsilon_step_size': [],\n",
    "    'best_model_data': ''\n",
    "}\n",
    "\n",
    "#load_data(loaded, 'batch_size_and_replay_steps')\n",
    "#load_data(loaded, 'e_greedy_and_softmax')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### COMPUTE AVERAGES ####\n",
    "compute_average(loaded['batch_size_and_replay_steps'])\n",
    "compute_average(loaded['softmax_tau_step_size'])\n",
    "compute_average(loaded['e_greedy_epsilon_step_size'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#### PLOT ALL GRAPHS FOR SAME TYPE OF TESTS #####\n",
    "\n",
    "plot_same_type('rewards', 'batch_size_and_replay_steps', 0)    \n",
    "#plot_same_type('rewards', 'softmax_tau_step_size', 0)    \n",
    "#plot_same_type('rewards', 'e_greedy_epsilon_step_size', 0)   \n",
    "\n",
    "#plot_same_type('loss', 'batch_size_and_replay_steps', 0)    \n",
    "#plot_same_type('loss', 'softmax_tau_step_size', 0)    \n",
    "#plot_same_type('loss', 'e_greedy_epsilon_step_size', 0)   \n",
    "\n",
    "#plot_same_type('episode_steps', 'batch_size_and_replay_steps', 0)    \n",
    "#plot_same_type('episode_steps', 'softmax_tau_step_size', 0)    \n",
    "#plot_same_type('episode_steps', 'e_greedy_epsilon_step_size', 0)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "######################### Choosing the best batch size and number of replay steps ############################\n",
    "batch_size_and_replay_steps_average_loss = [loaded['batch_size_and_replay_steps'][i]['average_loss'] for i in range(len(loaded['batch_size_and_replay_steps']))]\n",
    "batch_size_and_replay_steps_average_metric = [loaded['batch_size_and_replay_steps'][i]['average_metric'] for i in range(len(loaded['batch_size_and_replay_steps']))]\n",
    "batch_size_and_replay_steps_average_rewards = [loaded['batch_size_and_replay_steps'][i]['average_reward'] for i in range(len(loaded['batch_size_and_replay_steps']))]\n",
    "\n",
    "\n",
    "print('batch_sizes: ', [8,16,32,64,128], ' replays_steps: ', [2,4,8,16])\n",
    "plot(batch_size_and_replay_steps_average_loss, [], '', 0, 30, 'Average Loss for Bach-Size and Replay-Steps Testing', 'Test Number', 'Average Loss', ylim=True, ylim_val=[0, 10])\n",
    "#plot(batch_size_and_replay_steps_average_metric, [], '', 0, 30, 'Average Metric', 'Test Number', 'Average Metric')\n",
    "plot(batch_size_and_replay_steps_average_rewards, [], '', 0, 30, 'Average Reward for Bach-Size and Replay-Steps Testing', 'Test Number', 'Average Reward')\n",
    "\n",
    "print(batch_size_and_replay_steps_average_loss)\n",
    "# Best config: batch size : 8 , replay steps : 8 (test number 2) with average reward of 144.92691799377076\n",
    "#.....  maybe it's not the best because 300 episodes are not enough to decide\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "######################### Choosing the best tau and step size in Softmax exploration ############################\n",
    "softmax_tau_step_size_average_loss = [loaded['softmax_tau_step_size'][i]['average_loss'] for i in range(len(loaded['softmax_tau_step_size']))]\n",
    "softmax_tau_step_size_average_metric = [loaded['softmax_tau_step_size'][i]['average_metric'] for i in range(len(loaded['softmax_tau_step_size']))]\n",
    "softmax_tau_step_size_average_rewards = [loaded['softmax_tau_step_size'][i]['average_reward'] for i in range(len(loaded['softmax_tau_step_size']))]\n",
    "\n",
    "print('tau_values: ', [0.001, 0.01, 0.1, 1.0], ' step_sizes: ', [0.0001, 0.0005, 0.001, 0.01, 0.05, 0.1])\n",
    "plot(softmax_tau_step_size_average_loss, [], '', 0, 30, 'Average Loss for Tau and Step-Size Testing', 'Test Number', 'Average Loss', ylim=True, ylim_val=[0, 10])\n",
    "#plot(softmax_tau_step_size_average_metric, [], '', 0, 30, 'Average Metric', 'Test Number', 'Average Metric')\n",
    "plot(softmax_tau_step_size_average_rewards, [], '', 0, 30, 'Average Reward for Tau and Step-Size Testing', 'Test Number', 'Average Reward')\n",
    "\n",
    "print(softmax_tau_step_size_average_loss)\n",
    "# Best config: tau : 0.01 , step size : 0.0005 (test number 6) with average reward of 120.65607773858783\n",
    "#.....  maybe it's not the best because 300 episodes are not enough to decide"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "######################### Choosing the best e_greedy_M and step size in e_greedy exploration ############################\n",
    "e_greedy_epsilon_step_size_average_loss = [loaded['e_greedy_epsilon_step_size'][i]['average_loss'] for i in range(len(loaded['e_greedy_epsilon_step_size']))]\n",
    "e_greedy_epsilon_step_size_average_metric = [loaded['e_greedy_epsilon_step_size'][i]['average_metric'] for i in range(len(loaded['e_greedy_epsilon_step_size']))]\n",
    "e_greedy_epsilon_step_size_average_rewards = [loaded['e_greedy_epsilon_step_size'][i]['average_reward'] for i in range(len(loaded['e_greedy_epsilon_step_size']))]\n",
    "\n",
    "print('e_greedy_m_values: ', [20, 50, 100], ' step_sizes: ', [0.0001, 0.0005, 0.001, 0.01, 0.1])\n",
    "plot(e_greedy_epsilon_step_size_average_loss, [], '', 0, 30, 'Average Loss for Epsilon and Step-Size Testing', 'Test Number', 'Average Loss', ylim=True, ylim_val=[0, 10])\n",
    "#plot(e_greedy_epsilon_step_size_average_metric, [], '', 0, 30, 'Average Metric', 'Test Number', 'Average Metric')\n",
    "plot(e_greedy_epsilon_step_size_average_rewards, [], '', 0, 30, 'Average Reward for Epsilon and Step-Size Testing', 'Test Number', 'Average Reward')\n",
    "\n",
    "print(e_greedy_epsilon_step_size_average_loss)\n",
    "# Best config e greedy M : 50 , step size : 0.001 (test number 7) with average reward of 89.85369241704876\n",
    "#.....  maybe it's not because 300 episodes are not enough to decide"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "###### MUST READ THIS PLEASE ############  IMPORTANT  ############\n",
    "# We see that the average reward of Softmax exploration (120.65607773858783) is greater than e_greedy\n",
    "# exploration (89.85369241704876), so we will do the training again using Softmax exploration \n",
    "# with these parameters :\n",
    "# batch size : 8 , replay steps : 8\n",
    "# tau : 0.01 , step size : 0.0005\n",
    "# NOTE: these tests that we did were running simultaneously on 2 different computers and 2 different google colab \n",
    "# servers (that means 4 computers) took about full 3 days to complete. we did other tests than these but some \n",
    "# of them weren't structured enough and others were not completed because google colab servers were constantly \n",
    "# failing and our computers weren't enough. we could have gotten much better and precise results if we \n",
    "# extended the test period to 500 or 600 episodes.\n",
    "# NOTE: this is all we could do. we really wanted to include more parameters in the tests but it would have taken \n",
    "# much longer, so we chose the parameters that we knew they were the most important and impactful on learning process\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The training cell is above\n",
    "# Load data and model\n",
    "load_data(loaded, 'best_model_data')\n",
    "# Reconstruct the Neural network\n",
    "network = reconstruct_model('.\\\\results\\\\best_model\\\\model_softmax.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "############################ PRINT BEST MODEL DATA ##############################\n",
    "#plot(loaded['best_model_data']['loss'], [], '', 1, 100000, 'Loss', 'Learning Curve', 'Loss')\n",
    "#plot(loaded['best_model_data']['episode_steps'], [], '', 1, 30, 'number of steps', 'Learning Curve', 'episode steps')\n",
    "plot(loaded['best_model_data']['rewards'], [], '', 1, 100, 'Sum of rewards for each episode using a moving average', 'Episodes', 'Reward', clear=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make('LunarLander-v2')\n",
    "\n",
    "for _ in range(5):\n",
    "    state = env.reset()\n",
    "    for _ in range(300):\n",
    "        env.render()\n",
    "        state_tensor = tf.Variable(np.array([state]), dtype=tf.float32)\n",
    "        action_values_tensor = network(state_tensor)\n",
    "        action_values_proto_tensor = tf.make_tensor_proto(action_values_tensor)\n",
    "        action_values = tf.make_ndarray(action_values_proto_tensor)\n",
    "        action = argmax(action_values, np.random)\n",
    "        print('action :', action)\n",
    "        state, reward, is_terminal, _ = env.step(action) # take a random action\n",
    "    \n",
    "env.close()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}