07. Reinforcement Learning/2. Thompson Sampling.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Ad 1</th>\n",
       "      <th>Ad 2</th>\n",
       "      <th>Ad 3</th>\n",
       "      <th>Ad 4</th>\n",
       "      <th>Ad 5</th>\n",
       "      <th>Ad 6</th>\n",
       "      <th>Ad 7</th>\n",
       "      <th>Ad 8</th>\n",
       "      <th>Ad 9</th>\n",
       "      <th>Ad 10</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Ad 1  Ad 2  Ad 3  Ad 4  Ad 5  Ad 6  Ad 7  Ad 8  Ad 9  Ad 10\n",
       "0     1     0     0     0     1     0     0     0     1      0\n",
       "1     0     0     0     0     0     0     0     0     1      0\n",
       "2     0     0     0     0     0     0     0     0     0      0\n",
       "3     0     1     0     0     0     0     0     1     0      0\n",
       "4     0     0     0     0     0     0     0     0     0      0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "ds = pd.read_csv('Ads_CTR_Optimisation.csv')\n",
    "ds.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# total number of ads\n",
    "d = 10\n",
    "# number of times the ad i got reward 1\n",
    "number_of_rewards_1 = [0] * d\n",
    "# number of times the ad i got reward 0\n",
    "number_of_rewards_0 = [0] * d\n",
    "# total number of rounds\n",
    "N = 10000\n",
    "# the ads algorithm will select\n",
    "ads_selected = []\n",
    "# total reward\n",
    "total_reward = 0\n",
    "# number of times the ad i was selected\n",
    "numbers_of_selection = [0] * d\n",
    "# for all rounds\n",
    "for n in range(0, N):\n",
    "    max_random = 0\n",
    "    ad = 0\n",
    "    # for all ads\n",
    "    for i in range(0, d):\n",
    "        random_beta = random.betavariate(number_of_rewards_1[i] + 1,\n",
    "                                        number_of_rewards_0[i] + 1)\n",
    "        if random_beta > max_random:\n",
    "            max_random = random_beta\n",
    "            ad = i\n",
    "    ads_selected.append(ad)\n",
    "    reward = ds.values[n, ad]\n",
    "    total_reward += reward\n",
    "    if reward == 1:\n",
    "        number_of_rewards_1[ad] = number_of_rewards_1[ad] + 1\n",
    "    else:\n",
    "        number_of_rewards_0[ad] = number_of_rewards_0[ad] + 1\n",
    "    numbers_of_selection[ad] = numbers_of_selection[ad] + 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2638"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAD8CAYAAACcjGjIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAERhJREFUeJzt3X+s3XV9x/HnSyoiOGzBQrBlA2MD\noomCDVZJiKMKBY1liyRdNmlIty4LKhoTh/7TzB8JJsZfySRpKFocA1nV0KgTG8CZJQMpP1SwmnaI\ncKXSmiI6iT+q7/1xPt0OfG5Luaf3nKt9PpKb8/2+v5/veX9OW+7rfj/ney6pKiRJGvacSU9AkjT3\nGA6SpI7hIEnqGA6SpI7hIEnqGA6SpM4zhkOSa5PsSnL/UO24JFuSbG+PC1o9ST6VZEeS7yQ5a+ic\n1W389iSrh+qvTvLdds6nkuRQv0hJ0rNzMFcOnwVWPK12JXBrVS0Bbm37ABcCS9rXWuBqGIQJsA54\nDXA2sG5foLQxa4fOe3ovSdKYPWM4VNU3gT1PK68ENrbtjcDFQ/XrauAOYH6Sk4ALgC1VtaeqHge2\nACvasWOr6r9q8Gm864aeS5I0IfNmeN6JVbUToKp2Jjmh1RcBjwyNm2q1A9WnpqlPK8laBlcZHHPM\nMa8+/fTTZzh9STr83H333T+tqoUHM3am4bA/071fUDOoT6uq1gPrAZYuXVpbt26dyRwl6bCU5EcH\nO3amdys91paEaI+7Wn0KOHlo3GLg0WeoL56mLkmaoJmGw2Zg3x1Hq4Gbh+qXtruWlgFPtOWnW4Dz\nkyxob0SfD9zSjv0iybJ2l9KlQ88lSZqQZ1xWSnID8HrgRUmmGNx1dBVwU5I1wMPAJW34V4GLgB3A\nk8BlAFW1J8kHgbvauA9U1b43uf+BwR1Rzwf+vX1JkiYof6i/stv3HCTp2Ulyd1UtPZixfkJaktQx\nHCRJHcNBktQxHCRJHcNBktQ51J+QluacU678yqz3eOiqN816D2mcvHKQJHUMB0lSx3CQJHUMB0lS\nx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQ\nJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHUMB0lSx3CQJHVG\nCock707yQJL7k9yQ5Kgkpya5M8n2JJ9PcmQb+7y2v6MdP2Xoed7X6j9IcsFoL0mSNKoZh0OSRcA7\ngaVV9QrgCGAV8BHg41W1BHgcWNNOWQM8XlUvBT7expHkjHbey4EVwKeTHDHTeUmSRjfqstI84PlJ\n5gFHAzuB84BN7fhG4OK2vbLt044vT5JWv7Gqfl1VPwR2AGePOC9J0ghmHA5V9WPgo8DDDELhCeBu\n4GdVtbcNmwIWte1FwCPt3L1t/PHD9WnOeYoka5NsTbJ19+7dM526JOkZjLKstIDBT/2nAi8GjgEu\nnGZo7TtlP8f2V++LVeuramlVLV24cOGzn7Qk6aCMsqz0BuCHVbW7qn4LfBF4HTC/LTMBLAYebdtT\nwMkA7fgLgT3D9WnOkSRNwCjh8DCwLMnR7b2D5cD3gNuBt7Yxq4Gb2/bmtk87fltVVauvancznQos\nAb41wrwkSSOa98xDpldVdybZBNwD7AXuBdYDXwFuTPKhVtvQTtkAfC7JDgZXDKva8zyQ5CYGwbIX\nuLyqfjfTeUmSRjfjcACoqnXAuqeVH2Sau42q6lfAJft5ng8DHx5lLpKkQ8dPSEuSOoaDJKljOEiS\nOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaD\nJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKlj\nOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKkzUjgkmZ9kU5LvJ9mW5LVJjkuyJcn29rigjU2STyXZ\nkeQ7Sc4aep7Vbfz2JKtHfVGSpNGMeuXwSeBrVXU68EpgG3AlcGtVLQFubfsAFwJL2tda4GqAJMcB\n64DXAGcD6/YFiiRpMmYcDkmOBc4FNgBU1W+q6mfASmBjG7YRuLhtrwSuq4E7gPlJTgIuALZU1Z6q\nehzYAqyY6bwkSaMb5crhJcBu4DNJ7k1yTZJjgBOraidAezyhjV8EPDJ0/lSr7a/eSbI2ydYkW3fv\n3j3C1CVJBzJKOMwDzgKurqozgV/y/0tI08k0tTpAvS9Wra+qpVW1dOHChc92vpKkgzRKOEwBU1V1\nZ9vfxCAsHmvLRbTHXUPjTx46fzHw6AHqkqQJmXE4VNVPgEeSnNZKy4HvAZuBfXccrQZubtubgUvb\nXUvLgCfastMtwPlJFrQ3os9vNUnShMwb8fx3ANcnORJ4ELiMQeDclGQN8DBwSRv7VeAiYAfwZBtL\nVe1J8kHgrjbuA1W1Z8R5SZJGMFI4VNV9wNJpDi2fZmwBl+/nea4Frh1lLpKkQ8dPSEuSOoaDJKlj\nOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiS\nOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaD\nJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOoaDJKljOEiSOiOHQ5Ijktyb5Mtt/9QkdybZnuTzSY5s\n9ee1/R3t+ClDz/G+Vv9BkgtGnZMkaTSH4srhCmDb0P5HgI9X1RLgcWBNq68BHq+qlwIfb+NIcgaw\nCng5sAL4dJIjDsG8JEkzNFI4JFkMvAm4pu0HOA/Y1IZsBC5u2yvbPu348jZ+JXBjVf26qn4I7ADO\nHmVekqTRjHrl8AngvcDv2/7xwM+qam/bnwIWte1FwCMA7fgTbfz/1ac55ymSrE2yNcnW3bt3jzh1\nSdL+zDgckrwZ2FVVdw+Xpxlaz3DsQOc8tVi1vqqWVtXShQsXPqv5SpIO3rwRzj0HeEuSi4CjgGMZ\nXEnMTzKvXR0sBh5t46eAk4GpJPOAFwJ7hur7DJ8jSZqAGV85VNX7qmpxVZ3C4A3l26rqr4Hbgbe2\nYauBm9v25rZPO35bVVWrr2p3M50KLAG+NdN5SZJGN8qVw/78I3Bjkg8B9wIbWn0D8LkkOxhcMawC\nqKoHktwEfA/YC1xeVb+bhXlJkg7SIQmHqvoG8I22/SDT3G1UVb8CLtnP+R8GPnwo5iJJGp2fkJYk\ndQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwH\nSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLH\ncJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVLHcJAkdQwHSVJnxuGQ5OQktyfZluSBJFe0+nFJtiTZ\n3h4XtHqSfCrJjiTfSXLW0HOtbuO3J1k9+suSJI1ilCuHvcB7quplwDLg8iRnAFcCt1bVEuDWtg9w\nIbCkfa0FroZBmADrgNcAZwPr9gWKJGkyZhwOVbWzqu5p278AtgGLgJXAxjZsI3Bx214JXFcDdwDz\nk5wEXABsqao9VfU4sAVYMdN5SZJGd0jec0hyCnAmcCdwYlXthEGAACe0YYuAR4ZOm2q1/dWn67M2\nydYkW3fv3n0opi5JmsbI4ZDkBcAXgHdV1c8PNHSaWh2g3her1lfV0qpaunDhwmc/WUnSQRkpHJI8\nl0EwXF9VX2zlx9pyEe1xV6tPAScPnb4YePQAdUnShIxyt1KADcC2qvrY0KHNwL47jlYDNw/VL213\nLS0DnmjLTrcA5ydZ0N6IPr/VJEkTMm+Ec88B3gZ8N8l9rfZ+4CrgpiRrgIeBS9qxrwIXATuAJ4HL\nAKpqT5IPAne1cR+oqj0jzEuSNKIZh0NV/SfTv18AsHya8QVcvp/nuha4dqZzkSQdWn5CWpLUMRwk\nSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3D\nQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLU\nMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUMRwkSR3DQZLUmTfpCeyTZAXwSeAI4JqqumrCU5oV\np1z5lVnv8dBVb5r1HpL+uM2JcEhyBPDPwBuBKeCuJJur6nuTnZkOFUPx8OLf9x++OREOwNnAjqp6\nECDJjcBKYFbC4XD9h3u4vu5JmuSf+eH69+2f+aGRqhpLowNOInkrsKKq/rbtvw14TVW9/Wnj1gJr\n2+5pwA/GNMUXAT8dUy97H969J93f3n/cvf+sqhYezMC5cuWQaWpdalXVemD97E/nqZJsraql4+5r\n78Ov96T72/vw6n0gc+VupSng5KH9xcCjE5qLJB325ko43AUsSXJqkiOBVcDmCc9Jkg5bc2JZqar2\nJnk7cAuDW1mvraoHJjytYWNfyrL3Ydt70v3tfXj13q858Ya0JGlumSvLSpKkOcRwkCR1DIcDSHJt\nkl1J7p9A75OT3J5kW5IHklwxxt5HJflWkm+33v80rt5Dczgiyb1Jvjzmvg8l+W6S+5JsHXPv+Uk2\nJfl++3t/7Zj6ntZe776vnyd51zh6t/7vbv/O7k9yQ5Kjxtj7itb3gXG85um+pyQ5LsmWJNvb44LZ\nnsfBMBwO7LPAign13gu8p6peBiwDLk9yxph6/xo4r6peCbwKWJFk2Zh673MFsG3MPff586p61QTu\nPf8k8LWqOh14JWN6/VX1g/Z6XwW8GngS+NI4eidZBLwTWFpVr2BwQ8qqMfV+BfB3DH5DwyuBNydZ\nMsttP0v/PeVK4NaqWgLc2vYnznA4gKr6JrBnQr13VtU9bfsXDL5RLBpT76qq/2m7z21fY7tzIcli\n4E3ANePqOWlJjgXOBTYAVNVvqupnE5jKcuC/q+pHY+w5D3h+knnA0YzvM04vA+6oqierai/wH8Bf\nzGbD/XxPWQlsbNsbgYtncw4Hy3D4A5DkFOBM4M4x9jwiyX3ALmBLVY2tN/AJ4L3A78fYc58Cvp7k\n7vbrWsblJcBu4DNtOe2aJMeMsf8+q4AbxtWsqn4MfBR4GNgJPFFVXx9T+/uBc5Mcn+Ro4CKe+mHc\ncTmxqnbC4IdC4IQJzKFjOMxxSV4AfAF4V1X9fFx9q+p3bZlhMXB2uwSfdUneDOyqqrvH0W8a51TV\nWcCFDJbyzh1T33nAWcDVVXUm8EvGvLzQPoD6FuDfxthzAYOfnE8FXgwck+RvxtG7qrYBHwG2AF8D\nvs1gOVcYDnNakucyCIbrq+qLk5hDW9r4BuN77+Uc4C1JHgJuBM5L8i9j6k1VPdoedzFYdz97TK2n\ngKmhK7RNDMJinC4E7qmqx8bY8w3AD6tqd1X9Fvgi8LpxNa+qDVV1VlWdy2C5Z/u4eg95LMlJAO1x\n1wTm0DEc5qgkYbD+vK2qPjbm3guTzG/bz2fwH/D3x9G7qt5XVYur6hQGSxy3VdVYfpJMckySP9m3\nDZzPYOlh1lXVT4BHkpzWSsuZpV9ZfwB/xRiXlJqHgWVJjm7/5pczxhsRkpzQHv8U+EvG//ph8KuC\nVrft1cDNE5hDZ078+oy5KskNwOuBFyWZAtZV1YYxtT8HeBvw3bb2D/D+qvrqGHqfBGxs/xOm5wA3\nVdVYbymdkBOBLw2+RzEP+Neq+toY+78DuL4t7zwIXDauxm3N/Y3A34+rJ0BV3ZlkE3APgyWdexnv\nr5P4QpLjgd8Cl1fV47PZbLrvKcBVwE1J1jAIy0tmcw4Hy1+fIUnquKwkSeoYDpKkjuEgSeoYDpKk\njuEgSeoYDpKkjuEgSer8Lxg725jLAGJPAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f1e9bc52b10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.bar(range(1, 11), numbers_of_selection, align=\"center\")\n",
    "plt.xticks(range(1, 11))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[51, 141, 28, 61, 9517, 23, 22, 93, 25, 39]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "numbers_of_selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}