Mario Renforcment Learning AI in Python.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8239dcba",
   "metadata": {},
   "source": [
    "# Setting Up Mario"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "14ba3374",
   "metadata": {},
   "source": [
    "## Install Dependencies \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3bcb94dc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: gym_super_mario_bros==7.3.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (7.3.0)\n",
      "Requirement already satisfied: nes-py>=8.0.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from gym_super_mario_bros==7.3.0) (8.2.1)\n",
      "Requirement already satisfied: gym>=0.17.2 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from nes-py>=8.0.0->gym_super_mario_bros==7.3.0) (0.21.0)\n",
      "Requirement already satisfied: tqdm>=4.48.2 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from nes-py>=8.0.0->gym_super_mario_bros==7.3.0) (4.64.0)\n",
      "Requirement already satisfied: numpy>=1.18.5 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from nes-py>=8.0.0->gym_super_mario_bros==7.3.0) (1.23.1)\n",
      "Requirement already satisfied: pyglet<=1.5.21,>=1.4.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from nes-py>=8.0.0->gym_super_mario_bros==7.3.0) (1.5.21)\n",
      "Requirement already satisfied: cloudpickle>=1.2.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from gym>=0.17.2->nes-py>=8.0.0->gym_super_mario_bros==7.3.0) (2.1.0)\n",
      "Requirement already satisfied: colorama in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from tqdm>=4.48.2->nes-py>=8.0.0->gym_super_mario_bros==7.3.0) (0.4.5)\n",
      "Requirement already satisfied: nes-py in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (8.2.1)\n",
      "Requirement already satisfied: numpy>=1.18.5 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from nes-py) (1.23.1)\n",
      "Requirement already satisfied: gym>=0.17.2 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from nes-py) (0.21.0)\n",
      "Requirement already satisfied: pyglet<=1.5.21,>=1.4.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from nes-py) (1.5.21)\n",
      "Requirement already satisfied: tqdm>=4.48.2 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from nes-py) (4.64.0)\n",
      "Requirement already satisfied: cloudpickle>=1.2.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from gym>=0.17.2->nes-py) (2.1.0)\n",
      "Requirement already satisfied: colorama in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from tqdm>=4.48.2->nes-py) (0.4.5)\n",
      "Requirement already satisfied: pytouch in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (0.4.2)\n",
      "Requirement already satisfied: boto3 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytouch) (1.24.46)\n",
      "Requirement already satisfied: omegaconf>=2.0.2 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytouch) (2.2.2)\n",
      "Requirement already satisfied: pytorch-lightning in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytouch) (1.7.0)\n",
      "Requirement already satisfied: onnx in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytouch) (1.12.0)\n",
      "Requirement already satisfied: open3d in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytouch) (0.15.1)\n",
      "Requirement already satisfied: torchmetrics in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytouch) (0.9.3)\n",
      "Requirement already satisfied: pandas>=1.0.3 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytouch) (1.4.3)\n",
      "Requirement already satisfied: attrdict in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytouch) (2.0.1)\n",
      "Requirement already satisfied: black in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytouch) (22.6.0)\n",
      "Requirement already satisfied: torchvision>=0.5.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytouch) (0.13.1)\n",
      "Requirement already satisfied: pytorch3d in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytouch) (0.0.1)\n",
      "Requirement already satisfied: onnxruntime in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytouch) (1.12.1)\n",
      "Requirement already satisfied: nox>=2019.11.9 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytouch) (2022.8.7)\n",
      "Requirement already satisfied: scipy in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytouch) (1.9.0)\n",
      "Requirement already satisfied: matplotlib>=3.2.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytouch) (3.5.2)\n",
      "Requirement already satisfied: flake8 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytouch) (5.0.4)\n",
      "Requirement already satisfied: flake8-copyright in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytouch) (0.2.3)\n",
      "Requirement already satisfied: hydra-core>=1.0.3 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytouch) (1.2.0)\n",
      "Requirement already satisfied: isort in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytouch) (5.10.1)\n",
      "Requirement already satisfied: opencv-python in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytouch) (4.6.0.66)\n",
      "Requirement already satisfied: numpy>=1.20.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytouch) (1.23.1)\n",
      "Requirement already satisfied: torch>=1.4.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytouch) (1.12.1)\n",
      "Requirement already satisfied: antlr4-python3-runtime==4.9.* in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from hydra-core>=1.0.3->pytouch) (4.9.3)\n",
      "Requirement already satisfied: packaging in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from hydra-core>=1.0.3->pytouch) (21.3)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from matplotlib>=3.2.1->pytouch) (4.34.4)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from matplotlib>=3.2.1->pytouch) (2.8.2)\n",
      "Requirement already satisfied: cycler>=0.10 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from matplotlib>=3.2.1->pytouch) (0.11.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from matplotlib>=3.2.1->pytouch) (1.4.4)\n",
      "Requirement already satisfied: pillow>=6.2.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from matplotlib>=3.2.1->pytouch) (9.2.0)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from matplotlib>=3.2.1->pytouch) (3.0.4)\n",
      "Requirement already satisfied: virtualenv>=14 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from nox>=2019.11.9->pytouch) (20.16.3)\n",
      "Requirement already satisfied: py<2.0.0,>=1.4 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from nox>=2019.11.9->pytouch) (1.11.0)\n",
      "Requirement already satisfied: colorlog<7.0.0,>=2.6.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from nox>=2019.11.9->pytouch) (6.6.0)\n",
      "Requirement already satisfied: argcomplete<3.0,>=1.9.4 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from nox>=2019.11.9->pytouch) (2.0.0)\n",
      "Requirement already satisfied: PyYAML>=5.1.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from omegaconf>=2.0.2->pytouch) (6.0)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pandas>=1.0.3->pytouch) (2022.1)\n",
      "Requirement already satisfied: typing-extensions in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from torch>=1.4.0->pytouch) (4.3.0)\n",
      "Requirement already satisfied: requests in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from torchvision>=0.5.0->pytouch) (2.28.1)\n",
      "Requirement already satisfied: six in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from attrdict->pytouch) (1.16.0)\n",
      "Requirement already satisfied: pathspec>=0.9.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from black->pytouch) (0.9.0)\n",
      "Requirement already satisfied: tomli>=1.1.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from black->pytouch) (2.0.1)\n",
      "Requirement already satisfied: platformdirs>=2 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from black->pytouch) (2.5.2)\n",
      "Requirement already satisfied: click>=8.0.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from black->pytouch) (8.1.3)\n",
      "Requirement already satisfied: mypy-extensions>=0.4.3 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from black->pytouch) (0.4.3)\n",
      "Requirement already satisfied: botocore<1.28.0,>=1.27.46 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from boto3->pytouch) (1.27.46)\n",
      "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from boto3->pytouch) (0.6.0)\n",
      "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from boto3->pytouch) (1.0.1)\n",
      "Requirement already satisfied: mccabe<0.8.0,>=0.7.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from flake8->pytouch) (0.7.0)\n",
      "Requirement already satisfied: pycodestyle<2.10.0,>=2.9.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from flake8->pytouch) (2.9.1)\n",
      "Requirement already satisfied: pyflakes<2.6.0,>=2.5.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from flake8->pytouch) (2.5.0)\n",
      "Requirement already satisfied: setuptools in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from flake8-copyright->pytouch) (61.2.0)\n",
      "Requirement already satisfied: protobuf<=3.20.1,>=3.12.2 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from onnx->pytouch) (3.19.4)\n",
      "Requirement already satisfied: flatbuffers in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from onnxruntime->pytouch) (2.0)\n",
      "Requirement already satisfied: coloredlogs in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from onnxruntime->pytouch) (15.0.1)\n",
      "Requirement already satisfied: sympy in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from onnxruntime->pytouch) (1.10.1)\n",
      "Requirement already satisfied: jupyter-packaging~=0.10 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from open3d->pytouch) (0.12.2)\n",
      "Requirement already satisfied: pygments>=2.7.4 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from open3d->pytouch) (2.11.2)\n",
      "Requirement already satisfied: ipywidgets>=7.6.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from open3d->pytouch) (7.7.1)\n",
      "Requirement already satisfied: jupyterlab==3.*,>=3.0.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from open3d->pytouch) (3.4.4)\n",
      "Requirement already satisfied: wheel>=0.36.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from open3d->pytouch) (0.37.1)\n",
      "Requirement already satisfied: jupyter-core in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyterlab==3.*,>=3.0.0->open3d->pytouch) (4.10.0)\n",
      "Requirement already satisfied: jupyter-server~=1.16 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyterlab==3.*,>=3.0.0->open3d->pytouch) (1.18.1)\n",
      "Requirement already satisfied: ipython in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyterlab==3.*,>=3.0.0->open3d->pytouch) (8.4.0)\n",
      "Requirement already satisfied: jinja2>=2.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyterlab==3.*,>=3.0.0->open3d->pytouch) (3.0.3)\n",
      "Requirement already satisfied: notebook<7 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyterlab==3.*,>=3.0.0->open3d->pytouch) (6.4.12)\n",
      "Requirement already satisfied: jupyterlab-server~=2.10 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyterlab==3.*,>=3.0.0->open3d->pytouch) (2.12.0)\n",
      "Requirement already satisfied: tornado>=6.1.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyterlab==3.*,>=3.0.0->open3d->pytouch) (6.1)\n",
      "Requirement already satisfied: nbclassic in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyterlab==3.*,>=3.0.0->open3d->pytouch) (0.3.5)\n",
      "Requirement already satisfied: pyDeprecate>=0.3.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytorch-lightning->pytouch) (0.3.2)\n",
      "Requirement already satisfied: tqdm>=4.57.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytorch-lightning->pytouch) (4.64.0)\n",
      "Requirement already satisfied: tensorboard>=2.9.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytorch-lightning->pytouch) (2.9.1)\n",
      "Requirement already satisfied: fsspec[http]!=2021.06.0,>=2021.05.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pytorch-lightning->pytouch) (2022.7.1)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.25.4 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from botocore<1.28.0,>=1.27.46->boto3->pytouch) (1.26.11)\n",
      "Requirement already satisfied: colorama in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from click>=8.0.0->black->pytouch) (0.4.5)\n",
      "Requirement already satisfied: aiohttp in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning->pytouch) (3.8.1)\n",
      "Requirement already satisfied: ipykernel>=4.5.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from ipywidgets>=7.6.0->open3d->pytouch) (6.9.1)\n",
      "Requirement already satisfied: traitlets>=4.3.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from ipywidgets>=7.6.0->open3d->pytouch) (5.1.1)\n",
      "Requirement already satisfied: ipython-genutils~=0.2.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from ipywidgets>=7.6.0->open3d->pytouch) (0.2.0)\n",
      "Requirement already satisfied: widgetsnbextension~=3.6.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from ipywidgets>=7.6.0->open3d->pytouch) (3.6.1)\n",
      "Requirement already satisfied: jupyterlab-widgets>=1.0.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from ipywidgets>=7.6.0->open3d->pytouch) (1.1.1)\n",
      "Requirement already satisfied: tomlkit in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyter-packaging~=0.10->open3d->pytouch) (0.11.2)\n",
      "Requirement already satisfied: deprecation in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyter-packaging~=0.10->open3d->pytouch) (2.1.0)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from tensorboard>=2.9.1->pytorch-lightning->pytouch) (2.10.0)\n",
      "Requirement already satisfied: absl-py>=0.4 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from tensorboard>=2.9.1->pytorch-lightning->pytouch) (1.2.0)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from tensorboard>=2.9.1->pytorch-lightning->pytouch) (2.2.1)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from tensorboard>=2.9.1->pytorch-lightning->pytouch) (0.4.6)\n",
      "Requirement already satisfied: grpcio>=1.24.3 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from tensorboard>=2.9.1->pytorch-lightning->pytouch) (1.47.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from tensorboard>=2.9.1->pytorch-lightning->pytouch) (3.4.1)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from tensorboard>=2.9.1->pytorch-lightning->pytouch) (1.8.1)\n",
      "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from tensorboard>=2.9.1->pytorch-lightning->pytouch) (0.6.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from requests->torchvision>=0.5.0->pytouch) (2022.6.15)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from requests->torchvision>=0.5.0->pytouch) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from requests->torchvision>=0.5.0->pytouch) (3.3)\n",
      "Requirement already satisfied: distlib<1,>=0.3.5 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from virtualenv>=14->nox>=2019.11.9->pytouch) (0.3.5)\n",
      "Requirement already satisfied: filelock<4,>=3.4.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from virtualenv>=14->nox>=2019.11.9->pytouch) (3.7.1)\n",
      "Requirement already satisfied: humanfriendly>=9.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from coloredlogs->onnxruntime->pytouch) (10.0)\n",
      "Requirement already satisfied: mpmath>=0.19 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from sympy->onnxruntime->pytouch) (1.2.1)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->pytorch-lightning->pytouch) (0.2.8)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->pytorch-lightning->pytouch) (4.9)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.9.1->pytorch-lightning->pytouch) (5.2.0)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.9.1->pytorch-lightning->pytouch) (1.3.1)\n",
      "Requirement already satisfied: pyreadline3 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from humanfriendly>=9.1->coloredlogs->onnxruntime->pytouch) (3.4.1)\n",
      "Requirement already satisfied: nest-asyncio in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from ipykernel>=4.5.1->ipywidgets>=7.6.0->open3d->pytouch) (1.5.5)\n",
      "Requirement already satisfied: debugpy<2.0,>=1.0.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from ipykernel>=4.5.1->ipywidgets>=7.6.0->open3d->pytouch) (1.5.1)\n",
      "Requirement already satisfied: matplotlib-inline<0.2.0,>=0.1.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from ipykernel>=4.5.1->ipywidgets>=7.6.0->open3d->pytouch) (0.1.2)\n",
      "Requirement already satisfied: jupyter-client<8.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from ipykernel>=4.5.1->ipywidgets>=7.6.0->open3d->pytouch) (7.2.2)\n",
      "Requirement already satisfied: stack-data in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from ipython->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (0.2.0)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from ipython->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (3.0.20)\n",
      "Requirement already satisfied: backcall in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from ipython->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (0.2.0)\n",
      "Requirement already satisfied: pickleshare in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from ipython->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (0.7.5)\n",
      "Requirement already satisfied: decorator in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from ipython->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (5.1.1)\n",
      "Requirement already satisfied: jedi>=0.16 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from ipython->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (0.18.1)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jinja2>=2.1->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (2.1.1)\n",
      "Requirement already satisfied: prometheus-client in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (0.14.1)\n",
      "Requirement already satisfied: nbconvert>=6.4.4 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (6.4.4)\n",
      "Requirement already satisfied: websocket-client in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (0.58.0)\n",
      "Requirement already satisfied: pywinpty in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (2.0.2)\n",
      "Requirement already satisfied: argon2-cffi in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (21.3.0)\n",
      "Requirement already satisfied: anyio<4,>=3.1.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (3.5.0)\n",
      "Requirement already satisfied: terminado>=0.8.3 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (0.13.1)\n",
      "Requirement already satisfied: pyzmq>=17 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (23.2.0)\n",
      "Requirement already satisfied: Send2Trash in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (1.8.0)\n",
      "Requirement already satisfied: nbformat>=5.2.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (5.3.0)\n",
      "Requirement already satisfied: pywin32>=1.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyter-core->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (302)\n",
      "Requirement already satisfied: jsonschema>=3.0.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyterlab-server~=2.10->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (4.4.0)\n",
      "Requirement already satisfied: babel in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyterlab-server~=2.10->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (2.9.1)\n",
      "Requirement already satisfied: json5 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyterlab-server~=2.10->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (0.9.6)\n",
      "Requirement already satisfied: entrypoints>=0.2.2 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jupyterlab-server~=2.10->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (0.4)\n",
      "Requirement already satisfied: importlib-metadata>=4.4 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from markdown>=2.6.8->tensorboard>=2.9.1->pytorch-lightning->pytouch) (4.12.0)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from aiohttp->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning->pytouch) (4.0.2)\n",
      "Requirement already satisfied: attrs>=17.3.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from aiohttp->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning->pytouch) (21.4.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from aiohttp->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning->pytouch) (6.0.2)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from aiohttp->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning->pytouch) (1.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from aiohttp->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning->pytouch) (1.3.1)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from aiohttp->fsspec[http]!=2021.06.0,>=2021.05.0->pytorch-lightning->pytouch) (1.8.1)\n",
      "Requirement already satisfied: sniffio>=1.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from anyio<4,>=3.1.0->jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (1.2.0)\n",
      "Requirement already satisfied: zipp>=0.5 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard>=2.9.1->pytorch-lightning->pytouch) (3.8.1)\n",
      "Requirement already satisfied: parso<0.9.0,>=0.8.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jedi>=0.16->ipython->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (0.8.3)\n",
      "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from jsonschema>=3.0.1->jupyterlab-server~=2.10->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (0.18.0)\n",
      "Requirement already satisfied: jupyterlab-pygments in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from nbconvert>=6.4.4->jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (0.1.2)\n",
      "Requirement already satisfied: nbclient<0.6.0,>=0.5.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from nbconvert>=6.4.4->jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (0.5.13)\n",
      "Requirement already satisfied: defusedxml in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from nbconvert>=6.4.4->jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (0.7.1)\n",
      "Requirement already satisfied: mistune<2,>=0.8.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from nbconvert>=6.4.4->jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (0.8.4)\n",
      "Requirement already satisfied: pandocfilters>=1.4.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from nbconvert>=6.4.4->jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (1.5.0)\n",
      "Requirement already satisfied: bleach in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from nbconvert>=6.4.4->jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (4.1.0)\n",
      "Requirement already satisfied: beautifulsoup4 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from nbconvert>=6.4.4->jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (4.11.1)\n",
      "Requirement already satisfied: testpath in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from nbconvert>=6.4.4->jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (0.6.0)\n",
      "Requirement already satisfied: fastjsonschema in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from nbformat>=5.2.0->jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (2.15.1)\n",
      "Requirement already satisfied: wcwidth in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0->ipython->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (0.2.5)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard>=2.9.1->pytorch-lightning->pytouch) (0.4.8)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.9.1->pytorch-lightning->pytouch) (3.2.0)\n",
      "Requirement already satisfied: argon2-cffi-bindings in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from argon2-cffi->jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (21.2.0)\n",
      "Requirement already satisfied: pure-eval in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from stack-data->ipython->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (0.2.2)\n",
      "Requirement already satisfied: executing in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from stack-data->ipython->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (0.8.3)\n",
      "Requirement already satisfied: asttokens in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from stack-data->ipython->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (2.0.5)\n",
      "Requirement already satisfied: cffi>=1.0.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from argon2-cffi-bindings->argon2-cffi->jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (1.15.1)\n",
      "Requirement already satisfied: soupsieve>1.2 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from beautifulsoup4->nbconvert>=6.4.4->jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (2.3.1)\n",
      "Requirement already satisfied: webencodings in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from bleach->nbconvert>=6.4.4->jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (0.5.1)\n",
      "Requirement already satisfied: pycparser in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from cffi>=1.0.1->argon2-cffi-bindings->argon2-cffi->jupyter-server~=1.16->jupyterlab==3.*,>=3.0.0->open3d->pytouch) (2.21)\n"
     ]
    }
   ],
   "source": [
    "!pip install gym_super_mario_bros==7.3.0\n",
    "!pip install nes-py \n",
    "!pip install pytouch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9de9f540",
   "metadata": {},
   "source": [
    "## Imports "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f138eda4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym_super_mario_bros\n",
    "from nes_py.wrappers import JoypadSpace\n",
    "from gym_super_mario_bros.actions import SIMPLE_MOVEMENT"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f072d08",
   "metadata": {},
   "source": [
    "## Buidling the enviroment\n",
    "Creates an enviroment and wraps the movment options into simple movment operations (256 movment operations down to 7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "f36d11f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym_super_mario_bros.make('SuperMarioBros-v0')\n",
    "env = JoypadSpace(env, SIMPLE_MOVEMENT)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db1508f6",
   "metadata": {},
   "source": [
    "## Running the Game\n",
    "A loop to run the game with time-out length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "d205aef0",
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[1;32mIn [24]\u001b[0m, in \u001b[0;36m<cell line: 2>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      4\u001b[0m     env\u001b[38;5;241m.\u001b[39mreset()\n\u001b[0;32m      5\u001b[0m   state, reward, done, info \u001b[38;5;241m=\u001b[39m env\u001b[38;5;241m.\u001b[39mstep(env\u001b[38;5;241m.\u001b[39maction_space\u001b[38;5;241m.\u001b[39msample()) \u001b[38;5;66;03m# Allows to pass through a movment operations to the game (Takes random actions at the moment)\u001b[39;00m\n\u001b[1;32m----> 6\u001b[0m   \u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrender\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m      7\u001b[0m env\u001b[38;5;241m.\u001b[39mclose\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\Mario-Reforcment_Learning\\lib\\site-packages\\gym\\core.py:295\u001b[0m, in \u001b[0;36mWrapper.render\u001b[1;34m(self, mode, **kwargs)\u001b[0m\n\u001b[0;32m    294\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrender\u001b[39m(\u001b[38;5;28mself\u001b[39m, mode\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhuman\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m--> 295\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39menv\u001b[38;5;241m.\u001b[39mrender(mode, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\Mario-Reforcment_Learning\\lib\\site-packages\\gym\\core.py:295\u001b[0m, in \u001b[0;36mWrapper.render\u001b[1;34m(self, mode, **kwargs)\u001b[0m\n\u001b[0;32m    294\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrender\u001b[39m(\u001b[38;5;28mself\u001b[39m, mode\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhuman\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m--> 295\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39menv\u001b[38;5;241m.\u001b[39mrender(mode, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\Mario-Reforcment_Learning\\lib\\site-packages\\nes_py\\nes_env.py:386\u001b[0m, in \u001b[0;36mNESEnv.render\u001b[1;34m(self, mode)\u001b[0m\n\u001b[0;32m    380\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mviewer \u001b[38;5;241m=\u001b[39m ImageViewer(\n\u001b[0;32m    381\u001b[0m             caption\u001b[38;5;241m=\u001b[39mcaption,\n\u001b[0;32m    382\u001b[0m             height\u001b[38;5;241m=\u001b[39mSCREEN_HEIGHT,\n\u001b[0;32m    383\u001b[0m             width\u001b[38;5;241m=\u001b[39mSCREEN_WIDTH,\n\u001b[0;32m    384\u001b[0m         )\n\u001b[0;32m    385\u001b[0m     \u001b[38;5;66;03m# show the screen on the image viewer\u001b[39;00m\n\u001b[1;32m--> 386\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mviewer\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mshow\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mscreen\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    387\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m mode \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mrgb_array\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[0;32m    388\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mscreen\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\Mario-Reforcment_Learning\\lib\\site-packages\\nes_py\\_image_viewer.py:136\u001b[0m, in \u001b[0;36mImageViewer.show\u001b[1;34m(self, frame)\u001b[0m\n\u001b[0;32m    134\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mopen()\n\u001b[0;32m    135\u001b[0m \u001b[38;5;66;03m# prepare the window for the next frame\u001b[39;00m\n\u001b[1;32m--> 136\u001b[0m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_window\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mclear\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    137\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_window\u001b[38;5;241m.\u001b[39mswitch_to()\n\u001b[0;32m    138\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_window\u001b[38;5;241m.\u001b[39mdispatch_events()\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\Mario-Reforcment_Learning\\lib\\site-packages\\pyglet\\window\\__init__.py:1348\u001b[0m, in \u001b[0;36mBaseWindow.clear\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1342\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mclear\u001b[39m(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m   1343\u001b[0m     \u001b[38;5;124;03m\"\"\"Clear the window.\u001b[39;00m\n\u001b[0;32m   1344\u001b[0m \n\u001b[0;32m   1345\u001b[0m \u001b[38;5;124;03m    This is a convenience method for clearing the color and depth\u001b[39;00m\n\u001b[0;32m   1346\u001b[0m \u001b[38;5;124;03m    buffer.  The window must be the active context (see `switch_to`).\u001b[39;00m\n\u001b[0;32m   1347\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[1;32m-> 1348\u001b[0m     \u001b[43mgl\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mglClear\u001b[49m\u001b[43m(\u001b[49m\u001b[43mgl\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mGL_COLOR_BUFFER_BIT\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m|\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mgl\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mGL_DEPTH_BUFFER_BIT\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\Mario-Reforcment_Learning\\lib\\site-packages\\pyglet\\gl\\lib.py:87\u001b[0m, in \u001b[0;36merrcheck\u001b[1;34m(result, func, arguments)\u001b[0m\n\u001b[0;32m     83\u001b[0m \u001b[38;5;28;01mclass\u001b[39;00m \u001b[38;5;21;01mGLException\u001b[39;00m(\u001b[38;5;167;01mException\u001b[39;00m):\n\u001b[0;32m     84\u001b[0m     \u001b[38;5;28;01mpass\u001b[39;00m\n\u001b[1;32m---> 87\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21merrcheck\u001b[39m(result, func, arguments):\n\u001b[0;32m     88\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m _debug_gl_trace:\n\u001b[0;32m     89\u001b[0m         \u001b[38;5;28;01mtry\u001b[39;00m:\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "done = True\n",
    "for step in range(100000):\n",
    "  if done:\n",
    "    env.reset()\n",
    "  state, reward, done, info = env.step(env.action_space.sample()) # Allows to pass through a movment operations to the game (Takes random actions at the moment)\n",
    "  env.render()\n",
    "env.close"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21ac6049",
   "metadata": {},
   "source": [
    "# Pre-Processing the Enviroment"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2cc50b6b",
   "metadata": {},
   "source": [
    "## Installing AI Dependancies "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3399c7e2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://download.pytorch.org/whl/cu113\n",
      "Requirement already satisfied: torch in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (1.12.1)\n",
      "Requirement already satisfied: torchvision in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (0.13.1)\n",
      "Requirement already satisfied: torchaudio in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (0.12.1+cu113)\n",
      "Requirement already satisfied: typing-extensions in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from torch) (4.3.0)\n",
      "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from torchvision) (9.2.0)\n",
      "Requirement already satisfied: numpy in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from torchvision) (1.23.1)\n",
      "Requirement already satisfied: requests in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from torchvision) (2.28.1)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from requests->torchvision) (2022.6.15)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from requests->torchvision) (3.3)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from requests->torchvision) (2.0.4)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from requests->torchvision) (1.26.11)\n",
      "Requirement already satisfied: stable-baselines3[extra] in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (1.6.0)\n",
      "Requirement already satisfied: pandas in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from stable-baselines3[extra]) (1.4.3)\n",
      "Requirement already satisfied: matplotlib in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from stable-baselines3[extra]) (3.5.2)\n",
      "Requirement already satisfied: torch>=1.11 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from stable-baselines3[extra]) (1.12.1)\n",
      "Requirement already satisfied: cloudpickle in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from stable-baselines3[extra]) (2.1.0)\n",
      "Requirement already satisfied: gym==0.21 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from stable-baselines3[extra]) (0.21.0)\n",
      "Requirement already satisfied: numpy in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from stable-baselines3[extra]) (1.23.1)\n",
      "Requirement already satisfied: ale-py==0.7.4 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from stable-baselines3[extra]) (0.7.4)\n",
      "Requirement already satisfied: pillow in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from stable-baselines3[extra]) (9.2.0)\n",
      "Requirement already satisfied: opencv-python in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from stable-baselines3[extra]) (4.6.0.66)\n",
      "Requirement already satisfied: tensorboard>=2.2.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from stable-baselines3[extra]) (2.9.1)\n",
      "Requirement already satisfied: autorom[accept-rom-license]~=0.4.2 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from stable-baselines3[extra]) (0.4.2)\n",
      "Requirement already satisfied: psutil in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from stable-baselines3[extra]) (5.9.1)\n",
      "Requirement already satisfied: protobuf~=3.19.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from stable-baselines3[extra]) (3.19.4)\n",
      "Requirement already satisfied: importlib-resources in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from ale-py==0.7.4->stable-baselines3[extra]) (5.9.0)\n",
      "Requirement already satisfied: importlib-metadata>=4.10.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from ale-py==0.7.4->stable-baselines3[extra]) (4.12.0)\n",
      "Requirement already satisfied: tqdm in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (4.64.0)\n",
      "Requirement already satisfied: click in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (8.1.3)\n",
      "Requirement already satisfied: requests in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (2.28.1)\n",
      "Requirement already satisfied: AutoROM.accept-rom-license in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (0.4.2)\n",
      "Requirement already satisfied: absl-py>=0.4 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.2.0)\n",
      "Requirement already satisfied: grpcio>=1.24.3 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.47.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (3.4.1)\n",
      "Requirement already satisfied: wheel>=0.26 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (0.37.1)\n",
      "Requirement already satisfied: werkzeug>=1.0.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (2.2.1)\n",
      "Requirement already satisfied: google-auth<3,>=1.6.3 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (2.10.0)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (0.4.6)\n",
      "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (0.6.1)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (1.8.1)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from tensorboard>=2.2.0->stable-baselines3[extra]) (61.2.0)\n",
      "Requirement already satisfied: typing-extensions in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from torch>=1.11->stable-baselines3[extra]) (4.3.0)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (21.3)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (3.0.4)\n",
      "Requirement already satisfied: cycler>=0.10 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (0.11.0)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (2.8.2)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (1.4.4)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from matplotlib->stable-baselines3[extra]) (4.34.4)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pandas->stable-baselines3[extra]) (2022.1)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (4.9)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (0.2.8)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (5.2.0)\n",
      "Requirement already satisfied: six>=1.9.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (1.16.0)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->stable-baselines3[extra]) (1.3.1)\n",
      "Requirement already satisfied: zipp>=0.5 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from importlib-metadata>=4.10.0->ale-py==0.7.4->stable-baselines3[extra]) (3.8.1)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (2.0.4)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (1.26.11)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (2022.6.15)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from requests->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (3.3)\n",
      "Requirement already satisfied: MarkupSafe>=2.1.1 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from werkzeug>=1.0.1->tensorboard>=2.2.0->stable-baselines3[extra]) (2.1.1)\n",
      "Requirement already satisfied: colorama in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from click->autorom[accept-rom-license]~=0.4.2->stable-baselines3[extra]) (0.4.5)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard>=2.2.0->stable-baselines3[extra]) (0.4.8)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in c:\\users\\abeel\\anaconda3\\envs\\mario-reforcment_learning\\lib\\site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard>=2.2.0->stable-baselines3[extra]) (3.2.0)\n"
     ]
    }
   ],
   "source": [
    "!pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu113\n",
    "!pip install stable-baselines3[extra]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bdcd3611",
   "metadata": {},
   "source": [
    "## Importing Wrappers for AI\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b977dec0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from gym.wrappers import GrayScaleObservation\n",
    "from stable_baselines3.common.vec_env import VecFrameStack, DummyVecEnv\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "4298e2f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create the base enviroment\n",
    "env = gym_super_mario_bros.make('SuperMarioBros-v0')\n",
    "\n",
    "# Simplify the controls \n",
    "env = JoypadSpace(env, SIMPLE_MOVEMENT)\n",
    "\n",
    "# GrayScale the enviroment\n",
    "env = GrayScaleObservation(env, keep_dim=True)\n",
    "\n",
    "# Wrap inside the dummy enviroment\n",
    "env = DummyVecEnv([lambda: env])\n",
    "\n",
    "# Stack the frames\n",
    "env = VecFrameStack(env, 4, channels_order='last')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2534cdae",
   "metadata": {},
   "source": [
    "# Building the RI Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "ff5ead86",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os \n",
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.common.callbacks import BaseCallback\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "c5347053",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TrainAndLoggingCallback(BaseCallback):\n",
    "\n",
    "    def __init__(self, check_freq, save_path, verbose=1):\n",
    "        super(TrainAndLoggingCallback, self).__init__(verbose)\n",
    "        self.check_freq = check_freq\n",
    "        self.save_path = save_path\n",
    "\n",
    "    def _init_callback(self):\n",
    "        if self.save_path is not None:\n",
    "            os.makedirs(self.save_path, exist_ok=True)\n",
    "\n",
    "    def _on_step(self):\n",
    "        if self.n_calls % self.check_freq == 0:\n",
    "            model_path = os.path.join(self.save_path, 'best_model_{}'.format(self.n_calls))\n",
    "            self.model.save(model_path)\n",
    "\n",
    "        return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "00c4a1cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "CHECKPOINT_DIR = './train/'\n",
    "LOG_DIR = './logs/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "f63dcbba",
   "metadata": {},
   "outputs": [],
   "source": [
    "callback = TrainAndLoggingCallback(check_freq=50000, save_path=CHECKPOINT_DIR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "947b541d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cpu device\n",
      "Wrapping the env in a VecTransposeImage.\n"
     ]
    }
   ],
   "source": [
    "model = PPO('CnnPolicy', env, verbose=1, tensorboard_log='./', learning_rate=0.000001, n_steps=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "0860d822",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logging to ./PPO_5\n",
      "----------------------------\n",
      "| time/              |     |\n",
      "|    fps             | 112 |\n",
      "|    iterations      | 1   |\n",
      "|    time_elapsed    | 4   |\n",
      "|    total_timesteps | 512 |\n",
      "----------------------------\n",
      "-------------------------------------------\n",
      "| time/                   |               |\n",
      "|    fps                  | 25            |\n",
      "|    iterations           | 2             |\n",
      "|    time_elapsed         | 40            |\n",
      "|    total_timesteps      | 1024          |\n",
      "| train/                  |               |\n",
      "|    approx_kl            | 3.1094998e-05 |\n",
      "|    clip_fraction        | 0             |\n",
      "|    clip_range           | 0.2           |\n",
      "|    entropy_loss         | -1.95         |\n",
      "|    explained_variance   | -0.00106      |\n",
      "|    learning_rate        | 1e-06         |\n",
      "|    loss                 | 127           |\n",
      "|    n_updates            | 10            |\n",
      "|    policy_gradient_loss | -0.000219     |\n",
      "|    value_loss           | 326           |\n",
      "-------------------------------------------\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[1;32mIn [19]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlearn\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtotal_timesteps\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m500000\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallback\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcallback\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\Mario-Reforcment_Learning\\lib\\site-packages\\stable_baselines3\\ppo\\ppo.py:310\u001b[0m, in \u001b[0;36mPPO.learn\u001b[1;34m(self, total_timesteps, callback, log_interval, eval_env, eval_freq, n_eval_episodes, tb_log_name, eval_log_path, reset_num_timesteps)\u001b[0m\n\u001b[0;32m    297\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mlearn\u001b[39m(\n\u001b[0;32m    298\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[0;32m    299\u001b[0m     total_timesteps: \u001b[38;5;28mint\u001b[39m,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    307\u001b[0m     reset_num_timesteps: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[0;32m    308\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPPO\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[1;32m--> 310\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlearn\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    311\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtotal_timesteps\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtotal_timesteps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    312\u001b[0m \u001b[43m        \u001b[49m\u001b[43mcallback\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcallback\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    313\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlog_interval\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlog_interval\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    314\u001b[0m \u001b[43m        \u001b[49m\u001b[43meval_env\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43meval_env\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    315\u001b[0m \u001b[43m        \u001b[49m\u001b[43meval_freq\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43meval_freq\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    316\u001b[0m \u001b[43m        \u001b[49m\u001b[43mn_eval_episodes\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mn_eval_episodes\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    317\u001b[0m \u001b[43m        \u001b[49m\u001b[43mtb_log_name\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtb_log_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    318\u001b[0m \u001b[43m        \u001b[49m\u001b[43meval_log_path\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43meval_log_path\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    319\u001b[0m \u001b[43m        \u001b[49m\u001b[43mreset_num_timesteps\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreset_num_timesteps\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    320\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\Mario-Reforcment_Learning\\lib\\site-packages\\stable_baselines3\\common\\on_policy_algorithm.py:267\u001b[0m, in \u001b[0;36mOnPolicyAlgorithm.learn\u001b[1;34m(self, total_timesteps, callback, log_interval, eval_env, eval_freq, n_eval_episodes, tb_log_name, eval_log_path, reset_num_timesteps)\u001b[0m\n\u001b[0;32m    264\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlogger\u001b[38;5;241m.\u001b[39mrecord(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtime/total_timesteps\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnum_timesteps, exclude\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtensorboard\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m    265\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlogger\u001b[38;5;241m.\u001b[39mdump(step\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnum_timesteps)\n\u001b[1;32m--> 267\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    269\u001b[0m callback\u001b[38;5;241m.\u001b[39mon_training_end()\n\u001b[0;32m    271\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\Mario-Reforcment_Learning\\lib\\site-packages\\stable_baselines3\\ppo\\ppo.py:270\u001b[0m, in \u001b[0;36mPPO.train\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    268\u001b[0m \u001b[38;5;66;03m# Optimization step\u001b[39;00m\n\u001b[0;32m    269\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpolicy\u001b[38;5;241m.\u001b[39moptimizer\u001b[38;5;241m.\u001b[39mzero_grad()\n\u001b[1;32m--> 270\u001b[0m \u001b[43mloss\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    271\u001b[0m \u001b[38;5;66;03m# Clip grad norm\u001b[39;00m\n\u001b[0;32m    272\u001b[0m th\u001b[38;5;241m.\u001b[39mnn\u001b[38;5;241m.\u001b[39mutils\u001b[38;5;241m.\u001b[39mclip_grad_norm_(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpolicy\u001b[38;5;241m.\u001b[39mparameters(), \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmax_grad_norm)\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\Mario-Reforcment_Learning\\lib\\site-packages\\torch\\_tensor.py:396\u001b[0m, in \u001b[0;36mTensor.backward\u001b[1;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[0;32m    387\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m has_torch_function_unary(\u001b[38;5;28mself\u001b[39m):\n\u001b[0;32m    388\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m handle_torch_function(\n\u001b[0;32m    389\u001b[0m         Tensor\u001b[38;5;241m.\u001b[39mbackward,\n\u001b[0;32m    390\u001b[0m         (\u001b[38;5;28mself\u001b[39m,),\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    394\u001b[0m         create_graph\u001b[38;5;241m=\u001b[39mcreate_graph,\n\u001b[0;32m    395\u001b[0m         inputs\u001b[38;5;241m=\u001b[39minputs)\n\u001b[1;32m--> 396\u001b[0m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mautograd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbackward\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgradient\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43minputs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\Mario-Reforcment_Learning\\lib\\site-packages\\torch\\autograd\\__init__.py:173\u001b[0m, in \u001b[0;36mbackward\u001b[1;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[0;32m    168\u001b[0m     retain_graph \u001b[38;5;241m=\u001b[39m create_graph\n\u001b[0;32m    170\u001b[0m \u001b[38;5;66;03m# The reason we repeat same the comment below is that\u001b[39;00m\n\u001b[0;32m    171\u001b[0m \u001b[38;5;66;03m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[0;32m    172\u001b[0m \u001b[38;5;66;03m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[1;32m--> 173\u001b[0m \u001b[43mVariable\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_execution_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrun_backward\u001b[49m\u001b[43m(\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[0;32m    174\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtensors\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mgrad_tensors_\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mretain_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcreate_graph\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    175\u001b[0m \u001b[43m    \u001b[49m\u001b[43mallow_unreachable\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43maccumulate_grad\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model.learn(total_timesteps=500000, callback=callback)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "6047a48e",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('final')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "893f5b59",
   "metadata": {},
   "source": [
    "# Testing it out\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "e1fdf86c",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = PPO.load('./train/best_model_500000')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "b6a2c24e",
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Input \u001b[1;32mIn [34]\u001b[0m, in \u001b[0;36m<cell line: 4>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m      4\u001b[0m     action, _state \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mpredict(state)\n\u001b[1;32m----> 5\u001b[0m     state, reward, done, info \u001b[38;5;241m=\u001b[39m \u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep\u001b[49m\u001b[43m(\u001b[49m\u001b[43maction\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m      6\u001b[0m     env\u001b[38;5;241m.\u001b[39mrender()\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\Mario-Reforcment_Learning\\lib\\site-packages\\stable_baselines3\\common\\vec_env\\base_vec_env.py:162\u001b[0m, in \u001b[0;36mVecEnv.step\u001b[1;34m(self, actions)\u001b[0m\n\u001b[0;32m    155\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m    156\u001b[0m \u001b[38;5;124;03mStep the environments with the given action\u001b[39;00m\n\u001b[0;32m    157\u001b[0m \n\u001b[0;32m    158\u001b[0m \u001b[38;5;124;03m:param actions: the action\u001b[39;00m\n\u001b[0;32m    159\u001b[0m \u001b[38;5;124;03m:return: observation, reward, done, information\u001b[39;00m\n\u001b[0;32m    160\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m    161\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstep_async(actions)\n\u001b[1;32m--> 162\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep_wait\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\Mario-Reforcment_Learning\\lib\\site-packages\\stable_baselines3\\common\\vec_env\\vec_frame_stack.py:48\u001b[0m, in \u001b[0;36mVecFrameStack.step_wait\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     44\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mstep_wait\u001b[39m(\n\u001b[0;32m     45\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[0;32m     46\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tuple[Union[np\u001b[38;5;241m.\u001b[39mndarray, Dict[\u001b[38;5;28mstr\u001b[39m, np\u001b[38;5;241m.\u001b[39mndarray]], np\u001b[38;5;241m.\u001b[39mndarray, np\u001b[38;5;241m.\u001b[39mndarray, List[Dict[\u001b[38;5;28mstr\u001b[39m, Any]],]:\n\u001b[1;32m---> 48\u001b[0m     observations, rewards, dones, infos \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvenv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep_wait\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     50\u001b[0m     observations, infos \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstackedobs\u001b[38;5;241m.\u001b[39mupdate(observations, dones, infos)\n\u001b[0;32m     52\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m observations, rewards, dones, infos\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\Mario-Reforcment_Learning\\lib\\site-packages\\stable_baselines3\\common\\vec_env\\dummy_vec_env.py:43\u001b[0m, in \u001b[0;36mDummyVecEnv.step_wait\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m     41\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mstep_wait\u001b[39m(\u001b[38;5;28mself\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m VecEnvStepReturn:\n\u001b[0;32m     42\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m env_idx \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnum_envs):\n\u001b[1;32m---> 43\u001b[0m         obs, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbuf_rews[env_idx], \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbuf_dones[env_idx], \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbuf_infos[env_idx] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menvs\u001b[49m\u001b[43m[\u001b[49m\u001b[43menv_idx\u001b[49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m     44\u001b[0m \u001b[43m            \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mactions\u001b[49m\u001b[43m[\u001b[49m\u001b[43menv_idx\u001b[49m\u001b[43m]\u001b[49m\n\u001b[0;32m     45\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     46\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbuf_dones[env_idx]:\n\u001b[0;32m     47\u001b[0m             \u001b[38;5;66;03m# save final observation where user can get it, then reset\u001b[39;00m\n\u001b[0;32m     48\u001b[0m             \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mbuf_infos[env_idx][\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mterminal_observation\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m obs\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\Mario-Reforcment_Learning\\lib\\site-packages\\gym\\core.py:323\u001b[0m, in \u001b[0;36mObservationWrapper.step\u001b[1;34m(self, action)\u001b[0m\n\u001b[0;32m    322\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mstep\u001b[39m(\u001b[38;5;28mself\u001b[39m, action):\n\u001b[1;32m--> 323\u001b[0m     observation, reward, done, info \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep\u001b[49m\u001b[43m(\u001b[49m\u001b[43maction\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    324\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mobservation(observation), reward, done, info\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\Mario-Reforcment_Learning\\lib\\site-packages\\nes_py\\wrappers\\joypad_space.py:74\u001b[0m, in \u001b[0;36mJoypadSpace.step\u001b[1;34m(self, action)\u001b[0m\n\u001b[0;32m     59\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m     60\u001b[0m \u001b[38;5;124;03mTake a step using the given action.\u001b[39;00m\n\u001b[0;32m     61\u001b[0m \n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m     71\u001b[0m \n\u001b[0;32m     72\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m     73\u001b[0m \u001b[38;5;66;03m# take the step and record the output\u001b[39;00m\n\u001b[1;32m---> 74\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_action_map\u001b[49m\u001b[43m[\u001b[49m\u001b[43maction\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\Mario-Reforcment_Learning\\lib\\site-packages\\gym\\wrappers\\time_limit.py:18\u001b[0m, in \u001b[0;36mTimeLimit.step\u001b[1;34m(self, action)\u001b[0m\n\u001b[0;32m     14\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mstep\u001b[39m(\u001b[38;5;28mself\u001b[39m, action):\n\u001b[0;32m     15\u001b[0m     \u001b[38;5;28;01massert\u001b[39;00m (\n\u001b[0;32m     16\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_elapsed_steps \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m     17\u001b[0m     ), \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCannot call env.step() before calling reset()\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m---> 18\u001b[0m     observation, reward, done, info \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43menv\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mstep\u001b[49m\u001b[43m(\u001b[49m\u001b[43maction\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     19\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_elapsed_steps \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[0;32m     20\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_elapsed_steps \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_max_episode_steps:\n",
      "File \u001b[1;32m~\\anaconda3\\envs\\Mario-Reforcment_Learning\\lib\\site-packages\\nes_py\\nes_env.py:300\u001b[0m, in \u001b[0;36mNESEnv.step\u001b[1;34m(self, action)\u001b[0m\n\u001b[0;32m    298\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcontrollers[\u001b[38;5;241m0\u001b[39m][:] \u001b[38;5;241m=\u001b[39m action\n\u001b[0;32m    299\u001b[0m \u001b[38;5;66;03m# pass the action to the emulator as an unsigned byte\u001b[39;00m\n\u001b[1;32m--> 300\u001b[0m \u001b[43m_LIB\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mStep\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_env\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    301\u001b[0m \u001b[38;5;66;03m# get the reward for this step\u001b[39;00m\n\u001b[0;32m    302\u001b[0m reward \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mfloat\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_get_reward())\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "state = env.reset()\n",
    "\n",
    "while True:\n",
    "    action, _state = model.predict(state)\n",
    "    state, reward, done, info = env.step(action)\n",
    "    env.render()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "cf793f26",
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "588a68a1",
   "metadata": {},
   "outputs": [],
   "source": [
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "068eed1f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}