q learning 2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "2926188f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "fba44e73",
   "metadata": {},
   "outputs": [],
   "source": [
    "matrix_size = 6\n",
    "M = np.matrix([\n",
    "       [-1, -1, -1, -1, 0, -1],\n",
    "       [-1, -1, -1, 0, -1, 100],\n",
    "       [-1, -1, -1, 0, -1, -1],\n",
    "       [-1, 0., 0., -1, 0., -1],\n",
    "       [0., -1, -1, 0., -1, 100],\n",
    "       [-1, 0., -1, -1, 0., 100]\n",
    "])\n",
    "gamma = 0.85"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "33873888",
   "metadata": {},
   "outputs": [],
   "source": [
    "Q = np.matrix(np.zeros([matrix_size, matrix_size]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "4bd6043f",
   "metadata": {},
   "outputs": [],
   "source": [
    "initial_state = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "131d48f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 2, 4], dtype=int64)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def available_actions(state):\n",
    "    current_state_row = M[state,]\n",
    "    available_action = np.where(current_state_row >= 0)[1]\n",
    "    return available_action\n",
    "\n",
    "available_action = available_actions(initial_state)\n",
    "available_action"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a7621254",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<function __main__.available_actions(state)>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "available_actions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "2cd10be7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample_next_action(available_actions_range):\n",
    "    next_action = int(np.random.choice(available_actions_range,1))\n",
    "    return next_action\n",
    "\n",
    "action = sample_next_action(available_action)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "03cf1276",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def update(current_state,action,gamma):\n",
    "    max_index = np.where(Q[action, ] == np.max(Q[action, ]))[1]\n",
    "    if max_index.shape[0] > 1:\n",
    "        max_index = int(np.random.choice(max_index , size=1))\n",
    "    else:\n",
    "        max_index = int(max_index)\n",
    "    max_value = Q[action, max_index]\n",
    "    Q[current_state,action] = M[current_state, action] + gamma * max_value\n",
    "    if (np.max(Q) > 0):\n",
    "        return(np.sum(Q / np.max(Q) * 100))\n",
    "    else:\n",
    "        return 0\n",
    "    \n",
    "update(initial_state,action,gamma)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "94926a14",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trained matrix :\n",
      "[[0.         0.         0.         0.         0.0085     0.        ]\n",
      " [0.         0.         0.         0.0072248  0.         0.01      ]\n",
      " [0.         0.         0.         0.007225   0.         0.        ]\n",
      " [0.         0.0085     0.00614108 0.         0.0085     0.        ]\n",
      " [0.007225   0.         0.         0.0072248  0.         0.01      ]\n",
      " [0.         0.0085     0.         0.         0.00849948 0.00999972]]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = []\n",
    "for i in range(1000):\n",
    "    current_state = np.random.randint(0, int(Q.shape[0]))\n",
    "    available_action = available_actions(current_state)\n",
    "    action = sample_next_action(available_action)\n",
    "    score = update(current_state,action,gamma)\n",
    "    scores.append(score)\n",
    "\n",
    "print('Trained matrix :')\n",
    "print(Q / np.max(Q * 100))\n",
    "current_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "5df35690",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Most efficient path : \n",
      "[3, 4, 5]\n"
     ]
    }
   ],
   "source": [
    "current_state = 3\n",
    "steps = [current_state]\n",
    "\n",
    "\n",
    "while current_state != 5 :\n",
    "    next_index_step = np.where(Q[current_state,] == np.max(Q[current_state , ]))[1]\n",
    "    if next_index_step.shape[0] > 1:\n",
    "        next_index_step = int(np.random.choice(next_index_step, size=1))\n",
    "    else:\n",
    "        next_index_step = int(next_index_step)\n",
    "    steps.append(next_index_step)\n",
    "    current_state = next_index_step\n",
    "\n",
    "print('Most efficient path : ')\n",
    "print(steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1dc63c78",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8f57c8c8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}