Notebooks/VRP - Reinforcement Learning .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "c62333b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from collections import defaultdict\n",
    "from typing import List\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "495190f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "class VRPEnvironment:\n",
    "    def __init__(self, num_vehicles, capacity, customers, distance_matrix):\n",
    "        self.num_vehicles = num_vehicles\n",
    "        self.capacity = capacity\n",
    "        self.customers = customers\n",
    "        self.distance_matrix = distance_matrix\n",
    "        self.reset()\n",
    "        \n",
    "    def reset(self):\n",
    "        self.vehicle_positions = [0] * self.num_vehicles\n",
    "        self.customer_states = [tuple(self.customers)] * self.num_vehicles\n",
    "        self.demands = [0] * self.num_vehicles\n",
    "        self.total_distance = 0\n",
    "        self.steps = 0\n",
    "        \n",
    "    def step(self, actions):\n",
    "        rewards = [0] * self.num_vehicles\n",
    "        next_customer_states = [None] * self.num_vehicles\n",
    "        for i, action in enumerate(actions):\n",
    "            if action < len(self.customer_states[i]):\n",
    "                customer = self.customer_states[i][action]\n",
    "                if self.demands[i] + customer[1] <= self.capacity:\n",
    "                    self.demands[i] += customer[1]\n",
    "                    rewards[i] -= self.distance_matrix[self.vehicle_positions[i]][customer[0]]\n",
    "                    self.total_distance += self.distance_matrix[self.vehicle_positions[i]][customer[0]]\n",
    "                    self.vehicle_positions[i] = customer[0]\n",
    "                    next_customer_states[i] = self.customer_states[i][:action] + self.customer_states[i][action+1:]\n",
    "        done = (len(next_customer_states[0]) == 0)\n",
    "        state = tuple(self.vehicle_positions + [self.demands[0]] + [tuple(x) for x in next_customer_states])\n",
    "        return state, sum(rewards), done\n",
    "    \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "ff9addf0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_vrp_agent(num_vehicles, capacity, customers, distance_matrix, num_episodes=1000, alpha=0.1, gamma=0.9, epsilon=0.1):\n",
    "    num_customers = len(customers)\n",
    "\n",
    "    # Initialize the Q-table with a shape that can hold all possible states and actions\n",
    "    max_num_states = num_customers ** num_vehicles\n",
    "    max_num_actions = capacity + 1\n",
    "    q_table = np.zeros((max_num_states, max_num_actions))\n",
    "\n",
    "    # Train the agent for the specified number of episodes\n",
    "    for episode in range(num_episodes):\n",
    "        # Reset the environment for each episode\n",
    "        state = (0,)*num_vehicles\n",
    "        total_reward = 0\n",
    "\n",
    "        # Loop over all customers and choose an action for each vehicle\n",
    "        for customer_index in range(num_customers):\n",
    "            for vehicle_index in range(num_vehicles):\n",
    "                # Convert the state to an index in the Q-table\n",
    "                state_index = sum([state[i] * (num_customers ** i) for i in range(num_vehicles)])\n",
    "\n",
    "                # Choose an action using epsilon-greedy exploration\n",
    "                if random.uniform(0, 1) < epsilon:\n",
    "                    action = random.randint(0, capacity)\n",
    "                else:\n",
    "                    action = np.argmax(q_table[state_index])\n",
    "\n",
    "                # Calculate the reward for the chosen action\n",
    "                customer = customers[customer_index]\n",
    "                if action >= customer[1]:\n",
    "                    reward = customer[1] - distance_matrix[state[vehicle_index]][customer_index]\n",
    "                else:\n",
    "                    reward = -100\n",
    "\n",
    "                # Convert the next state to an index in the Q-table\n",
    "                next_state = state[:vehicle_index] + (customer_index,) + state[vehicle_index+1:]\n",
    "                next_state_index = sum([next_state[i] * (num_customers ** i) for i in range(num_vehicles)])\n",
    "\n",
    "                # Update the Q-table using Q-learning\n",
    "                q_table[state_index][action] += alpha * (reward + gamma * np.max(q_table[next_state_index]) - q_table[state_index][action])\n",
    "                total_reward += reward\n",
    "\n",
    "                # Move to the next state\n",
    "                state = next_state\n",
    "\n",
    "        # Print the total reward for the episode\n",
    "        print(\"Episode:\", episode, \"Total reward:\", total_reward)\n",
    "\n",
    "    # Return the Q-table\n",
    "    return q_table\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "66fe1899",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_vrp_agent(q_table, num_vehicles, capacity, customers, distance_matrix, num_episodes=1000, alpha=0.1, gamma=0.9, epsilon=0.1):\n",
    "    total_distance = 0\n",
    "    # Loop through all vehicles\n",
    "    for vehicle_idx in range(num_vehicles):\n",
    "        load = 0\n",
    "        distance = 0\n",
    "        # Set the starting state for the current vehicle\n",
    "        state = 0  # Starting depot node\n",
    "        # Create a list of unvisited nodes (excluding the starting depot node)\n",
    "        unvisited_nodes = list(range(1, len(customers)))\n",
    "        # Loop until the route for the current vehicle is complete\n",
    "        while True:\n",
    "            # Choose an action for the current state using the Q-table\n",
    "            q_values = q_table[state]\n",
    "            valid_actions = np.nonzero(q_values)[0]\n",
    "            action = random.choice(valid_actions)\n",
    "            # Get the next state and reward based on the chosen action\n",
    "            next_state = action\n",
    "            reward = 0\n",
    "            if next_state in unvisited_nodes:\n",
    "                # Calculate the load and distance for the state transition\n",
    "                load += customers[next_state]\n",
    "                if load > capacity:\n",
    "                    reward = -1  # Exceeding capacity is penalized\n",
    "                    load -= customers[next_state]\n",
    "                    next_state = state  # Stay at the current state\n",
    "                else:\n",
    "                    reward = 1  # Visiting a customer is rewarded\n",
    "                    distance += distance_matrix[state][next_state]\n",
    "                    unvisited_nodes.remove(next_state)\n",
    "            else:\n",
    "                # Penalize visiting an already visited node\n",
    "                reward = -1\n",
    "                next_state = state  # Stay at the current state\n",
    "            # Update the Q-table for the current state and action\n",
    "            old_value = q_table[state, action]\n",
    "            next_max = np.max(q_table[next_state])\n",
    "            new_value = (1 - alpha) * old_value + alpha * (reward + gamma * next_max)\n",
    "            q_table[state, action] = new_value\n",
    "            # Move to the next state\n",
    "            state = next_state\n",
    "            # Check if the route is complete\n",
    "            if not unvisited_nodes:\n",
    "                # Return to the starting depot node\n",
    "                distance += distance_matrix[state][0]\n",
    "                break\n",
    "            # Check if the current state and action are invalid\n",
    "            if state >= len(distance_matrix) or next_state >= len(distance_matrix[state]):\n",
    "                distance = float('inf')\n",
    "                break\n",
    "        # Add the distance traveled by the current vehicle to the total distance\n",
    "        total_distance += distance\n",
    "    return total_distance\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "01b5a693",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_vehicles = 2\n",
    "capacity = 100\n",
    "customers = [(1, 20), (2, 30), (3, 50), (4, 10), (5, 40)]\n",
    "distance_matrix = [\n",
    "    [0, 10, 20, 5, 15],\n",
    "    [10, 0, 15, 20, 25],\n",
    "    [20, 15, 0, 10, 30],\n",
    "    [5, 20, 10, 0, 10],\n",
    "    [15, 25, 30, 10, 0],\n",
    "]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "332b0cdb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 0 Total reward: -1000\n",
      "Episode: 1 Total reward: -635\n",
      "Episode: 2 Total reward: -865\n",
      "Episode: 3 Total reward: -735\n",
      "Episode: 4 Total reward: -500\n",
      "Episode: 5 Total reward: -600\n",
      "Episode: 6 Total reward: -730\n",
      "Episode: 7 Total reward: -260\n",
      "Episode: 8 Total reward: -360\n",
      "Episode: 9 Total reward: -260\n",
      "Episode: 10 Total reward: -360\n",
      "Episode: 11 Total reward: -140\n",
      "Episode: 12 Total reward: -140\n",
      "Episode: 13 Total reward: -40\n",
      "Episode: 14 Total reward: 80\n",
      "Episode: 15 Total reward: -55\n",
      "Episode: 16 Total reward: 80\n",
      "Episode: 17 Total reward: 210\n",
      "Episode: 18 Total reward: 210\n",
      "Episode: 19 Total reward: 210\n",
      "Episode: 20 Total reward: 210\n",
      "Episode: 21 Total reward: 210\n",
      "Episode: 22 Total reward: 80\n",
      "Episode: 23 Total reward: 210\n",
      "Episode: 24 Total reward: 210\n",
      "Episode: 25 Total reward: 210\n",
      "Episode: 26 Total reward: 210\n",
      "Episode: 27 Total reward: 210\n",
      "Episode: 28 Total reward: 210\n",
      "Episode: 29 Total reward: 75\n",
      "Episode: 30 Total reward: 210\n",
      "Episode: 31 Total reward: 80\n",
      "Episode: 32 Total reward: 210\n",
      "Episode: 33 Total reward: 210\n",
      "Episode: 34 Total reward: 210\n",
      "Episode: 35 Total reward: 80\n",
      "Episode: 36 Total reward: 210\n",
      "Episode: 37 Total reward: 210\n",
      "Episode: 38 Total reward: -45\n",
      "Episode: 39 Total reward: 210\n",
      "Episode: 40 Total reward: 75\n",
      "Episode: 41 Total reward: 210\n",
      "Episode: 42 Total reward: 210\n",
      "Episode: 43 Total reward: 75\n",
      "Episode: 44 Total reward: 210\n",
      "Episode: 45 Total reward: 210\n",
      "Episode: 46 Total reward: 75\n",
      "Episode: 47 Total reward: 75\n",
      "Episode: 48 Total reward: 210\n",
      "Episode: 49 Total reward: 210\n",
      "Episode: 50 Total reward: 75\n",
      "Episode: 51 Total reward: 210\n",
      "Episode: 52 Total reward: 210\n",
      "Episode: 53 Total reward: 90\n",
      "Episode: 54 Total reward: 75\n",
      "Episode: 55 Total reward: 210\n",
      "Episode: 56 Total reward: 210\n",
      "Episode: 57 Total reward: 75\n",
      "Episode: 58 Total reward: 210\n",
      "Episode: 59 Total reward: 210\n",
      "Episode: 60 Total reward: 210\n",
      "Episode: 61 Total reward: 210\n",
      "Episode: 62 Total reward: 80\n",
      "Episode: 63 Total reward: 210\n",
      "Episode: 64 Total reward: 210\n",
      "Episode: 65 Total reward: 210\n",
      "Episode: 66 Total reward: 210\n",
      "Episode: 67 Total reward: 210\n",
      "Episode: 68 Total reward: 210\n",
      "Episode: 69 Total reward: 80\n",
      "Episode: 70 Total reward: 80\n",
      "Episode: 71 Total reward: 210\n",
      "Episode: 72 Total reward: 210\n",
      "Episode: 73 Total reward: 210\n",
      "Episode: 74 Total reward: 210\n",
      "Episode: 75 Total reward: 210\n",
      "Episode: 76 Total reward: 210\n",
      "Episode: 77 Total reward: 210\n",
      "Episode: 78 Total reward: 210\n",
      "Episode: 79 Total reward: 90\n",
      "Episode: 80 Total reward: 210\n",
      "Episode: 81 Total reward: 210\n",
      "Episode: 82 Total reward: 210\n",
      "Episode: 83 Total reward: 210\n",
      "Episode: 84 Total reward: -60\n",
      "Episode: 85 Total reward: 80\n",
      "Episode: 86 Total reward: 75\n",
      "Episode: 87 Total reward: 210\n",
      "Episode: 88 Total reward: 210\n",
      "Episode: 89 Total reward: 75\n",
      "Episode: 90 Total reward: 210\n",
      "Episode: 91 Total reward: 210\n",
      "Episode: 92 Total reward: 210\n",
      "Episode: 93 Total reward: 210\n",
      "Episode: 94 Total reward: 210\n",
      "Episode: 95 Total reward: 75\n",
      "Episode: 96 Total reward: 210\n",
      "Episode: 97 Total reward: 210\n",
      "Episode: 98 Total reward: 210\n",
      "Episode: 99 Total reward: 210\n",
      "Episode: 100 Total reward: 210\n",
      "Episode: 101 Total reward: 210\n",
      "Episode: 102 Total reward: 210\n",
      "Episode: 103 Total reward: 210\n",
      "Episode: 104 Total reward: 210\n",
      "Episode: 105 Total reward: 210\n",
      "Episode: 106 Total reward: 210\n",
      "Episode: 107 Total reward: 90\n",
      "Episode: 108 Total reward: 210\n",
      "Episode: 109 Total reward: 210\n",
      "Episode: 110 Total reward: 210\n",
      "Episode: 111 Total reward: 210\n",
      "Episode: 112 Total reward: -60\n",
      "Episode: 113 Total reward: 90\n",
      "Episode: 114 Total reward: 210\n",
      "Episode: 115 Total reward: 210\n",
      "Episode: 116 Total reward: 210\n",
      "Episode: 117 Total reward: 210\n",
      "Episode: 118 Total reward: 210\n",
      "Episode: 119 Total reward: 210\n",
      "Episode: 120 Total reward: 210\n",
      "Episode: 121 Total reward: 210\n",
      "Episode: 122 Total reward: 210\n",
      "Episode: 123 Total reward: 210\n",
      "Episode: 124 Total reward: 210\n",
      "Episode: 125 Total reward: 75\n",
      "Episode: 126 Total reward: 210\n",
      "Episode: 127 Total reward: 75\n",
      "Episode: 128 Total reward: 210\n",
      "Episode: 129 Total reward: 80\n",
      "Episode: 130 Total reward: 210\n",
      "Episode: 131 Total reward: 75\n",
      "Episode: 132 Total reward: 210\n",
      "Episode: 133 Total reward: 210\n",
      "Episode: 134 Total reward: 210\n",
      "Episode: 135 Total reward: 210\n",
      "Episode: 136 Total reward: 80\n",
      "Episode: 137 Total reward: 210\n",
      "Episode: 138 Total reward: 210\n",
      "Episode: 139 Total reward: 210\n",
      "Episode: 140 Total reward: 210\n",
      "Episode: 141 Total reward: 210\n",
      "Episode: 142 Total reward: 110\n",
      "Episode: 143 Total reward: 210\n",
      "Episode: 144 Total reward: 90\n",
      "Episode: 145 Total reward: 210\n",
      "Episode: 146 Total reward: 210\n",
      "Episode: 147 Total reward: 210\n",
      "Episode: 148 Total reward: 90\n",
      "Episode: 149 Total reward: 210\n",
      "Episode: 150 Total reward: 210\n",
      "Episode: 151 Total reward: 210\n",
      "Episode: 152 Total reward: 210\n",
      "Episode: 153 Total reward: 210\n",
      "Episode: 154 Total reward: -45\n",
      "Episode: 155 Total reward: 210\n",
      "Episode: 156 Total reward: 80\n",
      "Episode: 157 Total reward: 210\n",
      "Episode: 158 Total reward: 210\n",
      "Episode: 159 Total reward: 210\n",
      "Episode: 160 Total reward: 80\n",
      "Episode: 161 Total reward: 210\n",
      "Episode: 162 Total reward: 210\n",
      "Episode: 163 Total reward: 210\n",
      "Episode: 164 Total reward: 80\n",
      "Episode: 165 Total reward: 210\n",
      "Episode: 166 Total reward: 90\n",
      "Episode: 167 Total reward: -45\n",
      "Episode: 168 Total reward: 90\n",
      "Episode: 169 Total reward: 210\n",
      "Episode: 170 Total reward: 110\n",
      "Episode: 171 Total reward: 210\n",
      "Episode: 172 Total reward: 210\n",
      "Episode: 173 Total reward: 210\n",
      "Episode: 174 Total reward: 80\n",
      "Episode: 175 Total reward: 210\n",
      "Episode: 176 Total reward: 210\n",
      "Episode: 177 Total reward: 210\n",
      "Episode: 178 Total reward: 210\n",
      "Episode: 179 Total reward: 210\n",
      "Episode: 180 Total reward: 210\n",
      "Episode: 181 Total reward: 210\n",
      "Episode: 182 Total reward: 210\n",
      "Episode: 183 Total reward: 210\n",
      "Episode: 184 Total reward: 210\n",
      "Episode: 185 Total reward: 75\n",
      "Episode: 186 Total reward: 210\n",
      "Episode: 187 Total reward: 210\n",
      "Episode: 188 Total reward: 210\n",
      "Episode: 189 Total reward: 210\n",
      "Episode: 190 Total reward: 210\n",
      "Episode: 191 Total reward: 210\n",
      "Episode: 192 Total reward: 210\n",
      "Episode: 193 Total reward: 210\n",
      "Episode: 194 Total reward: 90\n",
      "Episode: 195 Total reward: 210\n",
      "Episode: 196 Total reward: 80\n",
      "Episode: 197 Total reward: 210\n",
      "Episode: 198 Total reward: 210\n",
      "Episode: 199 Total reward: 210\n",
      "Episode: 200 Total reward: 80\n",
      "Episode: 201 Total reward: 75\n",
      "Episode: 202 Total reward: 210\n",
      "Episode: 203 Total reward: 210\n",
      "Episode: 204 Total reward: 210\n",
      "Episode: 205 Total reward: 210\n",
      "Episode: 206 Total reward: 210\n",
      "Episode: 207 Total reward: 210\n",
      "Episode: 208 Total reward: 90\n",
      "Episode: 209 Total reward: 75\n",
      "Episode: 210 Total reward: 75\n",
      "Episode: 211 Total reward: 210\n",
      "Episode: 212 Total reward: 210\n",
      "Episode: 213 Total reward: 210\n",
      "Episode: 214 Total reward: 75\n",
      "Episode: 215 Total reward: 210\n",
      "Episode: 216 Total reward: 210\n",
      "Episode: 217 Total reward: 210\n",
      "Episode: 218 Total reward: 210\n",
      "Episode: 219 Total reward: 210\n",
      "Episode: 220 Total reward: 210\n",
      "Episode: 221 Total reward: 210\n",
      "Episode: 222 Total reward: 210\n",
      "Episode: 223 Total reward: 90\n",
      "Episode: 224 Total reward: 210\n",
      "Episode: 225 Total reward: 75\n",
      "Episode: 226 Total reward: 210\n",
      "Episode: 227 Total reward: 210\n",
      "Episode: 228 Total reward: 210\n",
      "Episode: 229 Total reward: 90\n",
      "Episode: 230 Total reward: 75\n",
      "Episode: 231 Total reward: 210\n",
      "Episode: 232 Total reward: 75\n",
      "Episode: 233 Total reward: 210\n",
      "Episode: 234 Total reward: 80\n",
      "Episode: 235 Total reward: 210\n",
      "Episode: 236 Total reward: 80\n",
      "Episode: 237 Total reward: 210\n",
      "Episode: 238 Total reward: 210\n",
      "Episode: 239 Total reward: 210\n",
      "Episode: 240 Total reward: 210\n",
      "Episode: 241 Total reward: 75\n",
      "Episode: 242 Total reward: 210\n",
      "Episode: 243 Total reward: 210\n",
      "Episode: 244 Total reward: 210\n",
      "Episode: 245 Total reward: 210\n",
      "Episode: 246 Total reward: 210\n",
      "Episode: 247 Total reward: 210\n",
      "Episode: 248 Total reward: 90\n",
      "Episode: 249 Total reward: 210\n",
      "Episode: 250 Total reward: 210\n",
      "Episode: 251 Total reward: 210\n",
      "Episode: 252 Total reward: 210\n",
      "Episode: 253 Total reward: 210\n",
      "Episode: 254 Total reward: 80\n",
      "Episode: 255 Total reward: 210\n",
      "Episode: 256 Total reward: 90\n",
      "Episode: 257 Total reward: 90\n",
      "Episode: 258 Total reward: 210\n",
      "Episode: 259 Total reward: 80\n",
      "Episode: 260 Total reward: -55\n",
      "Episode: 261 Total reward: 210\n",
      "Episode: 262 Total reward: 210\n",
      "Episode: 263 Total reward: 210\n",
      "Episode: 264 Total reward: 210\n",
      "Episode: 265 Total reward: 210\n",
      "Episode: 266 Total reward: 210\n",
      "Episode: 267 Total reward: 90\n",
      "Episode: 268 Total reward: 210\n",
      "Episode: 269 Total reward: 210\n",
      "Episode: 270 Total reward: 210\n",
      "Episode: 271 Total reward: 210\n",
      "Episode: 272 Total reward: 210\n",
      "Episode: 273 Total reward: 210\n",
      "Episode: 274 Total reward: 75\n",
      "Episode: 275 Total reward: 210\n",
      "Episode: 276 Total reward: 210\n",
      "Episode: 277 Total reward: 90\n",
      "Episode: 278 Total reward: 75\n",
      "Episode: 279 Total reward: 210\n",
      "Episode: 280 Total reward: 90\n",
      "Episode: 281 Total reward: 75\n",
      "Episode: 282 Total reward: 90\n",
      "Episode: 283 Total reward: 210\n",
      "Episode: 284 Total reward: 210\n",
      "Episode: 285 Total reward: 210\n",
      "Episode: 286 Total reward: 210\n",
      "Episode: 287 Total reward: 210\n",
      "Episode: 288 Total reward: 75\n",
      "Episode: 289 Total reward: 210\n",
      "Episode: 290 Total reward: 210\n",
      "Episode: 291 Total reward: 80\n",
      "Episode: 292 Total reward: 210\n",
      "Episode: 293 Total reward: 210\n",
      "Episode: 294 Total reward: 90\n",
      "Episode: 295 Total reward: 80\n",
      "Episode: 296 Total reward: 210\n",
      "Episode: 297 Total reward: 210\n",
      "Episode: 298 Total reward: 210\n",
      "Episode: 299 Total reward: 90\n",
      "Episode: 300 Total reward: 210\n",
      "Episode: 301 Total reward: 80\n",
      "Episode: 302 Total reward: 210\n",
      "Episode: 303 Total reward: 210\n",
      "Episode: 304 Total reward: 90\n",
      "Episode: 305 Total reward: 80\n",
      "Episode: 306 Total reward: 90\n",
      "Episode: 307 Total reward: 210\n",
      "Episode: 308 Total reward: 210\n",
      "Episode: 309 Total reward: 80\n",
      "Episode: 310 Total reward: 80\n",
      "Episode: 311 Total reward: 210\n",
      "Episode: 312 Total reward: -30\n",
      "Episode: 313 Total reward: 80\n",
      "Episode: 314 Total reward: 210\n",
      "Episode: 315 Total reward: 210\n",
      "Episode: 316 Total reward: 210\n",
      "Episode: 317 Total reward: 210\n",
      "Episode: 318 Total reward: 210\n",
      "Episode: 319 Total reward: 210\n",
      "Episode: 320 Total reward: 80\n",
      "Episode: 321 Total reward: 210\n",
      "Episode: 322 Total reward: 210\n",
      "Episode: 323 Total reward: 210\n",
      "Episode: 324 Total reward: 75\n",
      "Episode: 325 Total reward: 210\n",
      "Episode: 326 Total reward: 210\n",
      "Episode: 327 Total reward: 210\n",
      "Episode: 328 Total reward: 210\n",
      "Episode: 329 Total reward: 80\n",
      "Episode: 330 Total reward: 75\n",
      "Episode: 331 Total reward: 210\n",
      "Episode: 332 Total reward: 80\n",
      "Episode: 333 Total reward: 90\n",
      "Episode: 334 Total reward: 75\n",
      "Episode: 335 Total reward: 210\n",
      "Episode: 336 Total reward: 210\n",
      "Episode: 337 Total reward: 75\n",
      "Episode: 338 Total reward: 110\n",
      "Episode: 339 Total reward: 80\n",
      "Episode: 340 Total reward: 210\n",
      "Episode: 341 Total reward: 210\n",
      "Episode: 342 Total reward: 210\n",
      "Episode: 343 Total reward: 210\n",
      "Episode: 344 Total reward: 210\n",
      "Episode: 345 Total reward: 210\n",
      "Episode: 346 Total reward: 210\n",
      "Episode: 347 Total reward: 210\n",
      "Episode: 348 Total reward: 210\n",
      "Episode: 349 Total reward: 210\n",
      "Episode: 350 Total reward: 210\n",
      "Episode: 351 Total reward: 90\n",
      "Episode: 352 Total reward: 210\n",
      "Episode: 353 Total reward: 210\n",
      "Episode: 354 Total reward: 210\n",
      "Episode: 355 Total reward: 80\n",
      "Episode: 356 Total reward: 210\n",
      "Episode: 357 Total reward: 80\n",
      "Episode: 358 Total reward: 75\n",
      "Episode: 359 Total reward: 210\n",
      "Episode: 360 Total reward: 210\n",
      "Episode: 361 Total reward: 210\n",
      "Episode: 362 Total reward: 210\n",
      "Episode: 363 Total reward: 210\n",
      "Episode: 364 Total reward: 210\n",
      "Episode: 365 Total reward: 210\n",
      "Episode: 366 Total reward: 210\n",
      "Episode: 367 Total reward: 80\n",
      "Episode: 368 Total reward: 210\n",
      "Episode: 369 Total reward: -40\n",
      "Episode: 370 Total reward: 210\n",
      "Episode: 371 Total reward: 90\n",
      "Episode: 372 Total reward: 210\n",
      "Episode: 373 Total reward: 210\n",
      "Episode: 374 Total reward: 90\n",
      "Episode: 375 Total reward: 210\n",
      "Episode: 376 Total reward: 210\n",
      "Episode: 377 Total reward: 210\n",
      "Episode: 378 Total reward: 110\n",
      "Episode: 379 Total reward: 90\n",
      "Episode: 380 Total reward: 210\n",
      "Episode: 381 Total reward: 210\n",
      "Episode: 382 Total reward: 210\n",
      "Episode: 383 Total reward: 210\n",
      "Episode: 384 Total reward: 210\n",
      "Episode: 385 Total reward: 210\n",
      "Episode: 386 Total reward: -55\n",
      "Episode: 387 Total reward: 210\n",
      "Episode: 388 Total reward: 90\n",
      "Episode: 389 Total reward: 210\n",
      "Episode: 390 Total reward: 210\n",
      "Episode: 391 Total reward: 90\n",
      "Episode: 392 Total reward: 210\n",
      "Episode: 393 Total reward: 210\n",
      "Episode: 394 Total reward: 90\n",
      "Episode: 395 Total reward: 80\n",
      "Episode: 396 Total reward: 80\n",
      "Episode: 397 Total reward: 210\n",
      "Episode: 398 Total reward: 210\n",
      "Episode: 399 Total reward: 210\n",
      "Episode: 400 Total reward: 210\n",
      "Episode: 401 Total reward: 210\n",
      "Episode: 402 Total reward: 210\n",
      "Episode: 403 Total reward: 210\n",
      "Episode: 404 Total reward: 210\n",
      "Episode: 405 Total reward: 80\n",
      "Episode: 406 Total reward: 90\n",
      "Episode: 407 Total reward: 210\n",
      "Episode: 408 Total reward: 210\n",
      "Episode: 409 Total reward: 75\n",
      "Episode: 410 Total reward: 210\n",
      "Episode: 411 Total reward: 75\n",
      "Episode: 412 Total reward: 210\n",
      "Episode: 413 Total reward: 75\n",
      "Episode: 414 Total reward: 80\n",
      "Episode: 415 Total reward: 210\n",
      "Episode: 416 Total reward: 80\n",
      "Episode: 417 Total reward: 90\n",
      "Episode: 418 Total reward: 210\n",
      "Episode: 419 Total reward: 80\n",
      "Episode: 420 Total reward: 210\n",
      "Episode: 421 Total reward: 210\n",
      "Episode: 422 Total reward: 75\n",
      "Episode: 423 Total reward: 210\n",
      "Episode: 424 Total reward: 210\n",
      "Episode: 425 Total reward: 210\n",
      "Episode: 426 Total reward: 210\n",
      "Episode: 427 Total reward: 210\n",
      "Episode: 428 Total reward: 210\n",
      "Episode: 429 Total reward: 210\n",
      "Episode: 430 Total reward: 210\n",
      "Episode: 431 Total reward: 210\n",
      "Episode: 432 Total reward: 210\n",
      "Episode: 433 Total reward: 210\n",
      "Episode: 434 Total reward: 210\n",
      "Episode: 435 Total reward: 75\n",
      "Episode: 436 Total reward: 75\n",
      "Episode: 437 Total reward: 210\n",
      "Episode: 438 Total reward: 210\n",
      "Episode: 439 Total reward: 210\n",
      "Episode: 440 Total reward: 210\n",
      "Episode: 441 Total reward: 210\n",
      "Episode: 442 Total reward: 80\n",
      "Episode: 443 Total reward: 210\n",
      "Episode: 444 Total reward: 210\n",
      "Episode: 445 Total reward: 75\n",
      "Episode: 446 Total reward: 210\n",
      "Episode: 447 Total reward: 80\n",
      "Episode: 448 Total reward: 210\n",
      "Episode: 449 Total reward: 90\n",
      "Episode: 450 Total reward: 210\n",
      "Episode: 451 Total reward: 210\n",
      "Episode: 452 Total reward: 210\n",
      "Episode: 453 Total reward: 210\n",
      "Episode: 454 Total reward: -40\n",
      "Episode: 455 Total reward: 210\n",
      "Episode: 456 Total reward: 210\n",
      "Episode: 457 Total reward: 210\n",
      "Episode: 458 Total reward: 210\n",
      "Episode: 459 Total reward: 75\n",
      "Episode: 460 Total reward: 210\n",
      "Episode: 461 Total reward: 75\n",
      "Episode: 462 Total reward: 210\n",
      "Episode: 463 Total reward: 75\n",
      "Episode: 464 Total reward: 210\n",
      "Episode: 465 Total reward: 80\n",
      "Episode: 466 Total reward: 210\n",
      "Episode: 467 Total reward: 210\n",
      "Episode: 468 Total reward: 210\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 469 Total reward: 210\n",
      "Episode: 470 Total reward: 210\n",
      "Episode: 471 Total reward: 210\n",
      "Episode: 472 Total reward: 90\n",
      "Episode: 473 Total reward: 210\n",
      "Episode: 474 Total reward: 210\n",
      "Episode: 475 Total reward: 210\n",
      "Episode: 476 Total reward: 210\n",
      "Episode: 477 Total reward: 210\n",
      "Episode: 478 Total reward: 210\n",
      "Episode: 479 Total reward: 210\n",
      "Episode: 480 Total reward: 210\n",
      "Episode: 481 Total reward: 210\n",
      "Episode: 482 Total reward: 210\n",
      "Episode: 483 Total reward: 210\n",
      "Episode: 484 Total reward: 210\n",
      "Episode: 485 Total reward: 80\n",
      "Episode: 486 Total reward: 210\n",
      "Episode: 487 Total reward: 80\n",
      "Episode: 488 Total reward: -30\n",
      "Episode: 489 Total reward: 210\n",
      "Episode: 490 Total reward: 210\n",
      "Episode: 491 Total reward: 210\n",
      "Episode: 492 Total reward: 80\n",
      "Episode: 493 Total reward: 210\n",
      "Episode: 494 Total reward: 210\n",
      "Episode: 495 Total reward: 90\n",
      "Episode: 496 Total reward: 90\n",
      "Episode: 497 Total reward: 90\n",
      "Episode: 498 Total reward: 90\n",
      "Episode: 499 Total reward: 210\n",
      "Episode: 500 Total reward: 75\n",
      "Episode: 501 Total reward: 210\n",
      "Episode: 502 Total reward: 210\n",
      "Episode: 503 Total reward: 210\n",
      "Episode: 504 Total reward: 110\n",
      "Episode: 505 Total reward: 90\n",
      "Episode: 506 Total reward: 90\n",
      "Episode: 507 Total reward: 210\n",
      "Episode: 508 Total reward: 210\n",
      "Episode: 509 Total reward: 210\n",
      "Episode: 510 Total reward: 210\n",
      "Episode: 511 Total reward: 90\n",
      "Episode: 512 Total reward: 210\n",
      "Episode: 513 Total reward: 210\n",
      "Episode: 514 Total reward: 210\n",
      "Episode: 515 Total reward: 210\n",
      "Episode: 516 Total reward: 210\n",
      "Episode: 517 Total reward: 210\n",
      "Episode: 518 Total reward: -55\n",
      "Episode: 519 Total reward: 80\n",
      "Episode: 520 Total reward: 210\n",
      "Episode: 521 Total reward: 210\n",
      "Episode: 522 Total reward: 210\n",
      "Episode: 523 Total reward: 210\n",
      "Episode: 524 Total reward: 210\n",
      "Episode: 525 Total reward: 75\n",
      "Episode: 526 Total reward: 210\n",
      "Episode: 527 Total reward: 80\n",
      "Episode: 528 Total reward: 210\n",
      "Episode: 529 Total reward: 210\n",
      "Episode: 530 Total reward: 210\n",
      "Episode: 531 Total reward: 210\n",
      "Episode: 532 Total reward: 80\n",
      "Episode: 533 Total reward: 210\n",
      "Episode: 534 Total reward: 210\n",
      "Episode: 535 Total reward: 210\n",
      "Episode: 536 Total reward: 210\n",
      "Episode: 537 Total reward: 210\n",
      "Episode: 538 Total reward: 210\n",
      "Episode: 539 Total reward: 210\n",
      "Episode: 540 Total reward: 210\n",
      "Episode: 541 Total reward: 210\n",
      "Episode: 542 Total reward: 210\n",
      "Episode: 543 Total reward: 90\n",
      "Episode: 544 Total reward: 210\n",
      "Episode: 545 Total reward: 110\n",
      "Episode: 546 Total reward: 210\n",
      "Episode: 547 Total reward: 210\n",
      "Episode: 548 Total reward: 210\n",
      "Episode: 549 Total reward: 210\n",
      "Episode: 550 Total reward: 80\n",
      "Episode: 551 Total reward: 210\n",
      "Episode: 552 Total reward: 210\n",
      "Episode: 553 Total reward: 210\n",
      "Episode: 554 Total reward: 210\n",
      "Episode: 555 Total reward: 210\n",
      "Episode: 556 Total reward: 90\n",
      "Episode: 557 Total reward: 90\n",
      "Episode: 558 Total reward: 210\n",
      "Episode: 559 Total reward: 210\n",
      "Episode: 560 Total reward: 210\n",
      "Episode: 561 Total reward: 210\n",
      "Episode: 562 Total reward: 90\n",
      "Episode: 563 Total reward: 210\n",
      "Episode: 564 Total reward: 210\n",
      "Episode: 565 Total reward: 210\n",
      "Episode: 566 Total reward: 90\n",
      "Episode: 567 Total reward: 210\n",
      "Episode: 568 Total reward: 210\n",
      "Episode: 569 Total reward: 210\n",
      "Episode: 570 Total reward: 210\n",
      "Episode: 571 Total reward: 210\n",
      "Episode: 572 Total reward: 210\n",
      "Episode: 573 Total reward: 90\n",
      "Episode: 574 Total reward: -25\n",
      "Episode: 575 Total reward: 210\n",
      "Episode: 576 Total reward: 210\n",
      "Episode: 577 Total reward: 210\n",
      "Episode: 578 Total reward: 210\n",
      "Episode: 579 Total reward: 210\n",
      "Episode: 580 Total reward: 90\n",
      "Episode: 581 Total reward: 75\n",
      "Episode: 582 Total reward: 210\n",
      "Episode: 583 Total reward: 210\n",
      "Episode: 584 Total reward: 210\n",
      "Episode: 585 Total reward: 210\n",
      "Episode: 586 Total reward: 90\n",
      "Episode: 587 Total reward: 210\n",
      "Episode: 588 Total reward: 210\n",
      "Episode: 589 Total reward: 210\n",
      "Episode: 590 Total reward: 90\n",
      "Episode: 591 Total reward: 210\n",
      "Episode: 592 Total reward: 210\n",
      "Episode: 593 Total reward: 210\n",
      "Episode: 594 Total reward: 210\n",
      "Episode: 595 Total reward: 210\n",
      "Episode: 596 Total reward: 210\n",
      "Episode: 597 Total reward: 210\n",
      "Episode: 598 Total reward: 210\n",
      "Episode: 599 Total reward: 210\n",
      "Episode: 600 Total reward: 210\n",
      "Episode: 601 Total reward: 210\n",
      "Episode: 602 Total reward: 90\n",
      "Episode: 603 Total reward: 90\n",
      "Episode: 604 Total reward: 210\n",
      "Episode: 605 Total reward: 75\n",
      "Episode: 606 Total reward: 210\n",
      "Episode: 607 Total reward: 210\n",
      "Episode: 608 Total reward: 210\n",
      "Episode: 609 Total reward: 210\n",
      "Episode: 610 Total reward: 210\n",
      "Episode: 611 Total reward: 210\n",
      "Episode: 612 Total reward: 90\n",
      "Episode: 613 Total reward: 210\n",
      "Episode: 614 Total reward: 90\n",
      "Episode: 615 Total reward: 75\n",
      "Episode: 616 Total reward: 210\n",
      "Episode: 617 Total reward: 80\n",
      "Episode: 618 Total reward: 210\n",
      "Episode: 619 Total reward: 210\n",
      "Episode: 620 Total reward: 80\n",
      "Episode: 621 Total reward: 75\n",
      "Episode: 622 Total reward: 210\n",
      "Episode: 623 Total reward: 75\n",
      "Episode: 624 Total reward: 210\n",
      "Episode: 625 Total reward: 210\n",
      "Episode: 626 Total reward: 210\n",
      "Episode: 627 Total reward: 210\n",
      "Episode: 628 Total reward: 210\n",
      "Episode: 629 Total reward: 210\n",
      "Episode: 630 Total reward: 90\n",
      "Episode: 631 Total reward: 210\n",
      "Episode: 632 Total reward: 210\n",
      "Episode: 633 Total reward: 210\n",
      "Episode: 634 Total reward: 210\n",
      "Episode: 635 Total reward: 210\n",
      "Episode: 636 Total reward: 210\n",
      "Episode: 637 Total reward: 90\n",
      "Episode: 638 Total reward: 210\n",
      "Episode: 639 Total reward: 210\n",
      "Episode: 640 Total reward: 90\n",
      "Episode: 641 Total reward: 210\n",
      "Episode: 642 Total reward: 210\n",
      "Episode: 643 Total reward: 80\n",
      "Episode: 644 Total reward: 75\n",
      "Episode: 645 Total reward: 75\n",
      "Episode: 646 Total reward: 210\n",
      "Episode: 647 Total reward: 210\n",
      "Episode: 648 Total reward: 210\n",
      "Episode: 649 Total reward: 210\n",
      "Episode: 650 Total reward: 210\n",
      "Episode: 651 Total reward: 210\n",
      "Episode: 652 Total reward: 210\n",
      "Episode: 653 Total reward: 210\n",
      "Episode: 654 Total reward: 210\n",
      "Episode: 655 Total reward: 210\n",
      "Episode: 656 Total reward: 210\n",
      "Episode: 657 Total reward: 210\n",
      "Episode: 658 Total reward: 110\n",
      "Episode: 659 Total reward: 90\n",
      "Episode: 660 Total reward: 210\n",
      "Episode: 661 Total reward: 210\n",
      "Episode: 662 Total reward: 80\n",
      "Episode: 663 Total reward: 210\n",
      "Episode: 664 Total reward: 210\n",
      "Episode: 665 Total reward: 210\n",
      "Episode: 666 Total reward: 210\n",
      "Episode: 667 Total reward: 210\n",
      "Episode: 668 Total reward: 210\n",
      "Episode: 669 Total reward: 210\n",
      "Episode: 670 Total reward: 210\n",
      "Episode: 671 Total reward: 210\n",
      "Episode: 672 Total reward: 210\n",
      "Episode: 673 Total reward: 90\n",
      "Episode: 674 Total reward: 210\n",
      "Episode: 675 Total reward: 210\n",
      "Episode: 676 Total reward: 210\n",
      "Episode: 677 Total reward: 210\n",
      "Episode: 678 Total reward: 90\n",
      "Episode: 679 Total reward: 90\n",
      "Episode: 680 Total reward: 210\n",
      "Episode: 681 Total reward: 210\n",
      "Episode: 682 Total reward: 210\n",
      "Episode: 683 Total reward: 210\n",
      "Episode: 684 Total reward: 210\n",
      "Episode: 685 Total reward: 210\n",
      "Episode: 686 Total reward: 75\n",
      "Episode: 687 Total reward: 90\n",
      "Episode: 688 Total reward: -45\n",
      "Episode: 689 Total reward: 210\n",
      "Episode: 690 Total reward: 80\n",
      "Episode: 691 Total reward: -10\n",
      "Episode: 692 Total reward: 80\n",
      "Episode: 693 Total reward: 210\n",
      "Episode: 694 Total reward: 90\n",
      "Episode: 695 Total reward: 210\n",
      "Episode: 696 Total reward: 210\n",
      "Episode: 697 Total reward: 80\n",
      "Episode: 698 Total reward: 80\n",
      "Episode: 699 Total reward: 75\n",
      "Episode: 700 Total reward: 210\n",
      "Episode: 701 Total reward: 90\n",
      "Episode: 702 Total reward: 210\n",
      "Episode: 703 Total reward: -25\n",
      "Episode: 704 Total reward: -60\n",
      "Episode: 705 Total reward: 210\n",
      "Episode: 706 Total reward: 210\n",
      "Episode: 707 Total reward: 210\n",
      "Episode: 708 Total reward: 210\n",
      "Episode: 709 Total reward: 210\n",
      "Episode: 710 Total reward: 210\n",
      "Episode: 711 Total reward: 210\n",
      "Episode: 712 Total reward: 75\n",
      "Episode: 713 Total reward: 210\n",
      "Episode: 714 Total reward: 80\n",
      "Episode: 715 Total reward: 80\n",
      "Episode: 716 Total reward: 210\n",
      "Episode: 717 Total reward: 210\n",
      "Episode: 718 Total reward: 210\n",
      "Episode: 719 Total reward: 110\n",
      "Episode: 720 Total reward: 210\n",
      "Episode: 721 Total reward: 210\n",
      "Episode: 722 Total reward: 210\n",
      "Episode: 723 Total reward: 80\n",
      "Episode: 724 Total reward: 80\n",
      "Episode: 725 Total reward: 75\n",
      "Episode: 726 Total reward: 210\n",
      "Episode: 727 Total reward: 210\n",
      "Episode: 728 Total reward: 75\n",
      "Episode: 729 Total reward: 210\n",
      "Episode: 730 Total reward: 75\n",
      "Episode: 731 Total reward: 210\n",
      "Episode: 732 Total reward: 210\n",
      "Episode: 733 Total reward: 210\n",
      "Episode: 734 Total reward: 210\n",
      "Episode: 735 Total reward: 75\n",
      "Episode: 736 Total reward: 210\n",
      "Episode: 737 Total reward: 210\n",
      "Episode: 738 Total reward: 75\n",
      "Episode: 739 Total reward: 210\n",
      "Episode: 740 Total reward: 210\n",
      "Episode: 741 Total reward: 90\n",
      "Episode: 742 Total reward: 210\n",
      "Episode: 743 Total reward: 210\n",
      "Episode: 744 Total reward: 80\n",
      "Episode: 745 Total reward: 210\n",
      "Episode: 746 Total reward: 75\n",
      "Episode: 747 Total reward: 210\n",
      "Episode: 748 Total reward: 90\n",
      "Episode: 749 Total reward: 210\n",
      "Episode: 750 Total reward: 210\n",
      "Episode: 751 Total reward: 210\n",
      "Episode: 752 Total reward: 210\n",
      "Episode: 753 Total reward: 90\n",
      "Episode: 754 Total reward: 75\n",
      "Episode: 755 Total reward: 75\n",
      "Episode: 756 Total reward: 80\n",
      "Episode: 757 Total reward: -155\n",
      "Episode: 758 Total reward: 210\n",
      "Episode: 759 Total reward: 210\n",
      "Episode: 760 Total reward: 210\n",
      "Episode: 761 Total reward: 210\n",
      "Episode: 762 Total reward: 210\n",
      "Episode: 763 Total reward: 210\n",
      "Episode: 764 Total reward: 210\n",
      "Episode: 765 Total reward: 210\n",
      "Episode: 766 Total reward: 210\n",
      "Episode: 767 Total reward: 210\n",
      "Episode: 768 Total reward: 210\n",
      "Episode: 769 Total reward: 210\n",
      "Episode: 770 Total reward: -60\n",
      "Episode: 771 Total reward: 210\n",
      "Episode: 772 Total reward: 210\n",
      "Episode: 773 Total reward: 80\n",
      "Episode: 774 Total reward: 90\n",
      "Episode: 775 Total reward: 210\n",
      "Episode: 776 Total reward: 210\n",
      "Episode: 777 Total reward: 210\n",
      "Episode: 778 Total reward: 210\n",
      "Episode: 779 Total reward: 210\n",
      "Episode: 780 Total reward: 210\n",
      "Episode: 781 Total reward: -55\n",
      "Episode: 782 Total reward: 210\n",
      "Episode: 783 Total reward: 80\n",
      "Episode: 784 Total reward: 210\n",
      "Episode: 785 Total reward: 210\n",
      "Episode: 786 Total reward: 210\n",
      "Episode: 787 Total reward: 210\n",
      "Episode: 788 Total reward: 210\n",
      "Episode: 789 Total reward: 90\n",
      "Episode: 790 Total reward: 210\n",
      "Episode: 791 Total reward: 210\n",
      "Episode: 792 Total reward: 210\n",
      "Episode: 793 Total reward: 210\n",
      "Episode: 794 Total reward: 210\n",
      "Episode: 795 Total reward: 210\n",
      "Episode: 796 Total reward: 210\n",
      "Episode: 797 Total reward: 210\n",
      "Episode: 798 Total reward: 210\n",
      "Episode: 799 Total reward: 210\n",
      "Episode: 800 Total reward: 210\n",
      "Episode: 801 Total reward: 210\n",
      "Episode: 802 Total reward: 210\n",
      "Episode: 803 Total reward: 210\n",
      "Episode: 804 Total reward: 210\n",
      "Episode: 805 Total reward: 210\n",
      "Episode: 806 Total reward: 210\n",
      "Episode: 807 Total reward: 210\n",
      "Episode: 808 Total reward: -55\n",
      "Episode: 809 Total reward: 90\n",
      "Episode: 810 Total reward: 80\n",
      "Episode: 811 Total reward: 210\n",
      "Episode: 812 Total reward: 210\n",
      "Episode: 813 Total reward: 80\n",
      "Episode: 814 Total reward: 210\n",
      "Episode: 815 Total reward: 210\n",
      "Episode: 816 Total reward: 210\n",
      "Episode: 817 Total reward: 210\n",
      "Episode: 818 Total reward: 210\n",
      "Episode: 819 Total reward: 210\n",
      "Episode: 820 Total reward: 210\n",
      "Episode: 821 Total reward: 210\n",
      "Episode: 822 Total reward: 90\n",
      "Episode: 823 Total reward: -25\n",
      "Episode: 824 Total reward: 210\n",
      "Episode: 825 Total reward: 210\n",
      "Episode: 826 Total reward: 210\n",
      "Episode: 827 Total reward: 210\n",
      "Episode: 828 Total reward: 210\n",
      "Episode: 829 Total reward: 210\n",
      "Episode: 830 Total reward: 80\n",
      "Episode: 831 Total reward: -40\n",
      "Episode: 832 Total reward: 90\n",
      "Episode: 833 Total reward: 210\n",
      "Episode: 834 Total reward: 210\n",
      "Episode: 835 Total reward: 210\n",
      "Episode: 836 Total reward: 210\n",
      "Episode: 837 Total reward: 210\n",
      "Episode: 838 Total reward: 210\n",
      "Episode: 839 Total reward: 210\n",
      "Episode: 840 Total reward: 80\n",
      "Episode: 841 Total reward: 90\n",
      "Episode: 842 Total reward: 210\n",
      "Episode: 843 Total reward: 210\n",
      "Episode: 844 Total reward: 210\n",
      "Episode: 845 Total reward: 90\n",
      "Episode: 846 Total reward: 210\n",
      "Episode: 847 Total reward: 210\n",
      "Episode: 848 Total reward: 80\n",
      "Episode: 849 Total reward: 210\n",
      "Episode: 850 Total reward: 210\n",
      "Episode: 851 Total reward: 210\n",
      "Episode: 852 Total reward: 210\n",
      "Episode: 853 Total reward: 210\n",
      "Episode: 854 Total reward: 210\n",
      "Episode: 855 Total reward: 210\n",
      "Episode: 856 Total reward: 80\n",
      "Episode: 857 Total reward: 210\n",
      "Episode: 858 Total reward: 90\n",
      "Episode: 859 Total reward: 210\n",
      "Episode: 860 Total reward: 210\n",
      "Episode: 861 Total reward: 90\n",
      "Episode: 862 Total reward: 210\n",
      "Episode: 863 Total reward: 210\n",
      "Episode: 864 Total reward: 210\n",
      "Episode: 865 Total reward: 210\n",
      "Episode: 866 Total reward: 210\n",
      "Episode: 867 Total reward: 80\n",
      "Episode: 868 Total reward: 210\n",
      "Episode: 869 Total reward: 80\n",
      "Episode: 870 Total reward: 210\n",
      "Episode: 871 Total reward: 210\n",
      "Episode: 872 Total reward: 80\n",
      "Episode: 873 Total reward: 210\n",
      "Episode: 874 Total reward: 210\n",
      "Episode: 875 Total reward: 90\n",
      "Episode: 876 Total reward: 80\n",
      "Episode: 877 Total reward: 210\n",
      "Episode: 878 Total reward: 80\n",
      "Episode: 879 Total reward: 210\n",
      "Episode: 880 Total reward: 210\n",
      "Episode: 881 Total reward: 210\n",
      "Episode: 882 Total reward: 90\n",
      "Episode: 883 Total reward: 210\n",
      "Episode: 884 Total reward: 210\n",
      "Episode: 885 Total reward: 210\n",
      "Episode: 886 Total reward: 210\n",
      "Episode: 887 Total reward: 75\n",
      "Episode: 888 Total reward: 210\n",
      "Episode: 889 Total reward: 210\n",
      "Episode: 890 Total reward: 210\n",
      "Episode: 891 Total reward: 80\n",
      "Episode: 892 Total reward: 210\n",
      "Episode: 893 Total reward: 110\n",
      "Episode: 894 Total reward: 210\n",
      "Episode: 895 Total reward: 210\n",
      "Episode: 896 Total reward: 75\n",
      "Episode: 897 Total reward: 210\n",
      "Episode: 898 Total reward: 210\n",
      "Episode: 899 Total reward: 90\n",
      "Episode: 900 Total reward: 75\n",
      "Episode: 901 Total reward: 210\n",
      "Episode: 902 Total reward: 210\n",
      "Episode: 903 Total reward: 90\n",
      "Episode: 904 Total reward: 210\n",
      "Episode: 905 Total reward: 210\n",
      "Episode: 906 Total reward: 80\n",
      "Episode: 907 Total reward: 210\n",
      "Episode: 908 Total reward: 210\n",
      "Episode: 909 Total reward: 210\n",
      "Episode: 910 Total reward: 75\n",
      "Episode: 911 Total reward: 210\n",
      "Episode: 912 Total reward: 210\n",
      "Episode: 913 Total reward: 210\n",
      "Episode: 914 Total reward: 90\n",
      "Episode: 915 Total reward: 210\n",
      "Episode: 916 Total reward: 210\n",
      "Episode: 917 Total reward: 210\n",
      "Episode: 918 Total reward: 210\n",
      "Episode: 919 Total reward: 210\n",
      "Episode: 920 Total reward: 90\n",
      "Episode: 921 Total reward: 210\n",
      "Episode: 922 Total reward: 210\n",
      "Episode: 923 Total reward: 75\n",
      "Episode: 924 Total reward: 210\n",
      "Episode: 925 Total reward: 210\n",
      "Episode: 926 Total reward: 210\n",
      "Episode: 927 Total reward: 210\n",
      "Episode: 928 Total reward: -45\n",
      "Episode: 929 Total reward: 210\n",
      "Episode: 930 Total reward: 210\n",
      "Episode: 931 Total reward: 210\n",
      "Episode: 932 Total reward: 210\n",
      "Episode: 933 Total reward: 210\n",
      "Episode: 934 Total reward: 90\n",
      "Episode: 935 Total reward: 90\n",
      "Episode: 936 Total reward: 210\n",
      "Episode: 937 Total reward: 90\n",
      "Episode: 938 Total reward: 210\n",
      "Episode: 939 Total reward: 210\n",
      "Episode: 940 Total reward: 210\n",
      "Episode: 941 Total reward: 75\n",
      "Episode: 942 Total reward: 210\n",
      "Episode: 943 Total reward: 210\n",
      "Episode: 944 Total reward: 90\n",
      "Episode: 945 Total reward: 210\n",
      "Episode: 946 Total reward: 75\n",
      "Episode: 947 Total reward: 210\n",
      "Episode: 948 Total reward: 210\n",
      "Episode: 949 Total reward: 210\n",
      "Episode: 950 Total reward: 75\n",
      "Episode: 951 Total reward: 210\n",
      "Episode: 952 Total reward: 210\n",
      "Episode: 953 Total reward: 210\n",
      "Episode: 954 Total reward: 210\n",
      "Episode: 955 Total reward: 210\n",
      "Episode: 956 Total reward: 210\n",
      "Episode: 957 Total reward: 80\n",
      "Episode: 958 Total reward: 90\n",
      "Episode: 959 Total reward: 210\n",
      "Episode: 960 Total reward: 210\n",
      "Episode: 961 Total reward: 210\n",
      "Episode: 962 Total reward: 210\n",
      "Episode: 963 Total reward: 210\n",
      "Episode: 964 Total reward: 210\n",
      "Episode: 965 Total reward: 210\n",
      "Episode: 966 Total reward: 80\n",
      "Episode: 967 Total reward: 210\n",
      "Episode: 968 Total reward: 75\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode: 969 Total reward: 210\n",
      "Episode: 970 Total reward: 90\n",
      "Episode: 971 Total reward: 75\n",
      "Episode: 972 Total reward: 75\n",
      "Episode: 973 Total reward: 210\n",
      "Episode: 974 Total reward: 90\n",
      "Episode: 975 Total reward: 210\n",
      "Episode: 976 Total reward: 210\n",
      "Episode: 977 Total reward: 80\n",
      "Episode: 978 Total reward: 210\n",
      "Episode: 979 Total reward: 210\n",
      "Episode: 980 Total reward: 210\n",
      "Episode: 981 Total reward: 210\n",
      "Episode: 982 Total reward: 90\n",
      "Episode: 983 Total reward: 90\n",
      "Episode: 984 Total reward: 75\n",
      "Episode: 985 Total reward: 210\n",
      "Episode: 986 Total reward: 210\n",
      "Episode: 987 Total reward: 210\n",
      "Episode: 988 Total reward: 210\n",
      "Episode: 989 Total reward: 75\n",
      "Episode: 990 Total reward: 210\n",
      "Episode: 991 Total reward: 210\n",
      "Episode: 992 Total reward: 210\n",
      "Episode: 993 Total reward: 210\n",
      "Episode: 994 Total reward: 210\n",
      "Episode: 995 Total reward: 210\n",
      "Episode: 996 Total reward: 90\n",
      "Episode: 997 Total reward: 210\n",
      "Episode: 998 Total reward: 210\n",
      "Episode: 999 Total reward: 75\n"
     ]
    }
   ],
   "source": [
    "q_table = train_vrp_agent(num_vehicles, capacity, customers, distance_matrix, num_episodes=1000)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "d7f20668",
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "unsupported operand type(s) for +=: 'int' and 'tuple'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [93]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m total_distance \u001b[38;5;241m=\u001b[39m \u001b[43mtest_vrp_agent\u001b[49m\u001b[43m(\u001b[49m\u001b[43mq_table\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_vehicles\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcapacity\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcustomers\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdistance_matrix\u001b[49m\u001b[43m)\u001b[49m\n",
      "Input \u001b[0;32mIn [90]\u001b[0m, in \u001b[0;36mtest_vrp_agent\u001b[0;34m(q_table, num_vehicles, capacity, customers, distance_matrix, num_episodes, alpha, gamma, epsilon)\u001b[0m\n\u001b[1;32m     19\u001b[0m reward \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[1;32m     20\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m next_state \u001b[38;5;129;01min\u001b[39;00m unvisited_nodes:\n\u001b[1;32m     21\u001b[0m     \u001b[38;5;66;03m# Calculate the load and distance for the state transition\u001b[39;00m\n\u001b[0;32m---> 22\u001b[0m     load \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m customers[next_state]\n\u001b[1;32m     23\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m load \u001b[38;5;241m>\u001b[39m capacity:\n\u001b[1;32m     24\u001b[0m         reward \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m  \u001b[38;5;66;03m# Exceeding capacity is penalized\u001b[39;00m\n",
      "\u001b[0;31mTypeError\u001b[0m: unsupported operand type(s) for +=: 'int' and 'tuple'"
     ]
    }
   ],
   "source": [
    "total_distance = test_vrp_agent(q_table, num_vehicles, capacity, customers, distance_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2e47f9e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}