tic_explore_starts.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set()\n",
    "import progressbar\n",
    "from random import choices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "states={}\n",
    "rx=-5\n",
    "ro=5\n",
    "empty=0\n",
    "startstate=\"#########\"\n",
    "player=1\n",
    "df=0.93\n",
    "alpha=0.05\n",
    "l=[]\n",
    "k=0.03\n",
    "c=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def addkey(key,player):\n",
    "    key=key+player\n",
    "    states[key]=np.random.uniform(-0.01,0.01,9)\n",
    "    \n",
    "def checkvalidaction(key,act):\n",
    "    if key[act]=='#':\n",
    "        return 1\n",
    "    else:\n",
    "        return 0\n",
    "    \n",
    "def getnext(state,action,player):\n",
    "    #print(state,\" \",action,\" \",player,\"\\n\")\n",
    "    l=list(state)\n",
    "    l[int(action)]=player\n",
    "    state=''.join(l)\n",
    "    return state\n",
    "\n",
    "def getwinner(state):\n",
    "    l=list(state)\n",
    "    temp=[['#' for i in range(3)] for j in range(3)]\n",
    "    for i in range(3):\n",
    "        for j in range(3):\n",
    "            temp[i][j]=l[i*3+j]\n",
    "            #print(temp)\n",
    "    for i in range(3):\n",
    "        if temp[i][0]==temp[i][1] and temp[i][1]==temp[i][2] and temp[i][2]!='#':\n",
    "            return temp[i][0]\n",
    "        if temp[0][i]==temp[1][i] and temp[1][i]==temp[2][i] and temp[0][i]!='#':\n",
    "            return temp[0][i]\n",
    "        \n",
    "    if temp[0][0]==temp[1][1] and temp[1][1]==temp[2][2] and temp[1][1]!='#':\n",
    "        return temp[0][0];\n",
    "    else: \n",
    "        if temp[0][2]==temp[1][1] and temp[2][0]==temp[1][1] and temp[2][0]!='#':\n",
    "            return temp[2][0]\n",
    "    return 0\n",
    "\n",
    "def getreward(state):\n",
    "    winner=getwinner(state)\n",
    "    if winner=='X':\n",
    "        return 1\n",
    "    if winner=='O':\n",
    "        return -1\n",
    "    if '#' not in state:\n",
    "        return 0.5\n",
    "    return 0\n",
    "\n",
    "def gameover(state):\n",
    "    if '#' not in state:\n",
    "        return 1\n",
    "    if getwinner(state)!=0:\n",
    "        return 1\n",
    "    return 0\n",
    "\n",
    "def gameoverlast(state):\n",
    "    if '#' not in state:\n",
    "        return 1\n",
    "    return 0\n",
    " \n",
    "def switchplayers(pl):\n",
    "    if pl=='X':\n",
    "        return 'O'\n",
    "    else :\n",
    "        return 'X'\n",
    "    \n",
    "def updateqvalues(state,act,ns,rw,pl):\n",
    "    ex=0\n",
    "    if gameoverlast(ns):\n",
    "        ex=rw\n",
    "    else :\n",
    "        if pl=='X':\n",
    "            ex=rw+(df*np.amin(states[ns+switchplayers(pl)]))\n",
    "            #print(ex)\n",
    "        else :\n",
    "            ex=rw+(df*np.amax(states[ns+switchplayers(pl)]))\n",
    "    state=state+pl\n",
    "    #if alpha*(ex-states[state][act])>0.1 and alpha*(ex-states[state][act])<1:\n",
    "        #print(alpha*(ex-states[state][act]))\n",
    "    states[state][act]=states[state][act]+alpha*(ex-states[state][act])\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def chooseaction(key,player):\n",
    "    if key==startstate:\n",
    "        return int(random.random()*9)\n",
    "    ch=int(random.random()*2)\n",
    "    if ch==1:\n",
    "        while(1):\n",
    "            act=int(random.random()*9)\n",
    "            if checkvalidaction(key,act):\n",
    "                return act\n",
    "    else:\n",
    "        key_t=key+player\n",
    "        x=np.argsort(states[key_t])\n",
    "        x=np.flip(x)\n",
    "        for y in x:\n",
    "            if checkvalidaction(key,y):\n",
    "                return y  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getplacesfilled(state):\n",
    "    c=0\n",
    "    for x in state:\n",
    "        if x==\"#\":\n",
    "            c+=1\n",
    "    return 9-c\n",
    "\n",
    "def onlypos(state):\n",
    "    l=list(state)\n",
    "    return l.index('#')\n",
    "\n",
    "def getcdf(temp):\n",
    "    t=[]\n",
    "    for i in range(len(temp)):\n",
    "        su=0.0\n",
    "        for j in range(i):\n",
    "            su=su+temp[j]\n",
    "        t.append(su)\n",
    "    return t\n",
    "\n",
    "def decodecdf(act,temp):\n",
    "    for i in range(8,-1,-1):\n",
    "        if act>temp[i]:\n",
    "            return i\n",
    "    return 0\n",
    "\n",
    "def cact(key,player,k):\n",
    "    if getplacesfilled(key)==8:\n",
    "        return onlypos(key)\n",
    "    val=states[key+player]\n",
    "    lowest=0\n",
    "    if player=='X':\n",
    "        lowest=np.amin(val)\n",
    "    else :\n",
    "        val=val*-1\n",
    "        lowest=np.amin(val)\n",
    "    if lowest<0:\n",
    "        lowest=lowest*-1+0.1\n",
    "        val=val+lowest\n",
    "    val=np.array([k**x for x in val])\n",
    "    val=val/(np.sum(val))\n",
    "    temp=[int(1000*val[i]) for i in range(9)]\n",
    "    temp=getcdf(temp)\n",
    "    while(1):\n",
    "        act=int(random.random()*1000)\n",
    "        act=decodecdf(act,temp)\n",
    "        if checkvalidaction(key,int(act)):\n",
    "            return act\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def qlearn(k,maxgames):\n",
    "    games=0\n",
    "    player='X'\n",
    "    playerop='O'\n",
    "    player=int(random.random()*2)\n",
    "    if player==1:\n",
    "        player='X'\n",
    "    else :\n",
    "        player='O'\n",
    "    for i in progressbar.progressbar(range(maxgames)):\n",
    "        key=startstate\n",
    "        cplayer=player\n",
    "        while(23):            \n",
    "            if key+player not in states:\n",
    "                addkey(key,player)\n",
    "            action=cact(key,player,k)\n",
    "            nextstate=getnext(key,action,player)\n",
    "            reward=getreward(nextstate)\n",
    "            if nextstate+switchplayers(player) not in states:\n",
    "                addkey(nextstate,switchplayers(player))\n",
    "            updateqvalues(key,action,nextstate,reward,player)\n",
    "            if gameover(nextstate)==1:\n",
    "                player=switchplayers(cplayer)\n",
    "                break\n",
    "            else :\n",
    "                key=nextstate\n",
    "                player=switchplayers(player)\n",
    "        if games%10000==0:\n",
    "            k+=0.1\n",
    "    #return k\n",
    "                        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98% (138466 of 140000) |############### | Elapsed Time: 0:01:22 ETA:   0:00:01"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-7-42f9e36f20c1>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mk\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m0.15\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0mstates\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mqlearn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m140000\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-6-3229e3c45440>\u001b[0m in \u001b[0;36mqlearn\u001b[1;34m(k, maxgames)\u001b[0m\n\u001b[0;32m     14\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mplayer\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mstates\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     15\u001b[0m                 \u001b[0maddkey\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mplayer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 16\u001b[1;33m             \u001b[0maction\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcact\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mplayer\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     17\u001b[0m             \u001b[0mnextstate\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mgetnext\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0maction\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mplayer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     18\u001b[0m             \u001b[0mreward\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mgetreward\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnextstate\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-5-ce2352f1f05e>\u001b[0m in \u001b[0;36mcact\u001b[1;34m(key, player, k)\u001b[0m\n\u001b[0;32m     44\u001b[0m     \u001b[1;32mwhile\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     45\u001b[0m         \u001b[0mact\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrandom\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mrandom\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m*\u001b[0m\u001b[1;36m1000\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 46\u001b[1;33m         \u001b[0mact\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdecodecdf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mact\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtemp\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     47\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcheckvalidaction\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mact\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     48\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mact\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-5-ce2352f1f05e>\u001b[0m in \u001b[0;36mdecodecdf\u001b[1;34m(act, temp)\u001b[0m\n\u001b[0;32m     21\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mdecodecdf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mact\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mtemp\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     22\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m8\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 23\u001b[1;33m         \u001b[1;32mif\u001b[0m \u001b[0mact\u001b[0m\u001b[1;33m>\u001b[0m\u001b[0mtemp\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     24\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     25\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "k=0.15\n",
    "states={}\n",
    "qlearn(k,140000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(states))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "def getdata():\n",
    "    with open(\"dataqes.pkl\",\"rb\")as f:\n",
    "        return pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getactfromuser(state):\n",
    "    while(1):\n",
    "        y=int(input(\"ROW\"))\n",
    "        x=int(input(\"COLUMN\"))\n",
    "        if checkvalidaction(state,(y-1)*3+(x-1)):\n",
    "            state=getnext(state,(y-1)*3+(x-1),'O')\n",
    "            return state\n",
    "\n",
    "def getactfromcomp(state):\n",
    "    x=np.argsort(states[state+'X'])\n",
    "    x=np.flip(x)\n",
    "    for y in x:\n",
    "        if checkvalidaction(state,y):\n",
    "            state=getnext(state,y,'X')\n",
    "            return state\n",
    "        \n",
    "def printboard(state):\n",
    "    l=list(state)\n",
    "    for i in range(3):\n",
    "        for j in range(3):\n",
    "            print(l[i*3+j],\" \",end='')\n",
    "        print(\"\\n\")\n",
    "        \n",
    "import os\n",
    "from IPython.display import clear_output\n",
    "def gameplay():\n",
    "    state=startstate\n",
    "    for i in range(5):\n",
    "        state=getactfromuser(state)\n",
    "        clear_output()\n",
    "        printboard(state)\n",
    "        print(states[state+'X'])\n",
    "        if getwinner(state)=='O':\n",
    "            print(\"You Win\\n\")\n",
    "            return 1\n",
    "        if gameover(state):\n",
    "            return 1\n",
    "        state=getactfromcomp(state)\n",
    "        #clear_output()\n",
    "        printboard(state)\n",
    "        #print(states[state+'O'])\n",
    "        if getwinner(state)=='X':\n",
    "            print(\"Computer wins\\n\")\n",
    "            return 1\n",
    "        \n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "O  #  O  \n",
      "\n",
      "#  O  #  \n",
      "\n",
      "X  #  X  \n",
      "\n",
      "[-3.06143903e-03 -2.38263148e-04  9.49166439e-03 -3.98284688e-04\n",
      " -2.34616132e-03 -2.74186358e-02  5.67002103e-03  8.51732948e-01\n",
      " -6.97139453e-03]\n",
      "O  #  O  \n",
      "\n",
      "#  O  #  \n",
      "\n",
      "X  X  X  \n",
      "\n",
      "Computer wins\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gameplay()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}