HindiMovieReviews_HF.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "HindiMovieReviews-HF.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "u0v6eJOBLwmE",
        "colab_type": "code",
        "outputId": "aa7b012b-b23d-449a-bd4b-0e32c97cda47",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "! pip install --upgrade pandas tensorflow_gpu ktrain"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already up-to-date: pandas in /usr/local/lib/python3.6/dist-packages (1.0.3)\n",
            "Collecting tensorflow_gpu\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0a/93/c7bca39b23aae45cd2e85ad3871c81eccc63b9c5276e926511e2e5b0879d/tensorflow_gpu-2.1.0-cp36-cp36m-manylinux2010_x86_64.whl (421.8MB)\n",
            "\u001b[K     |████████████████████████████████| 421.8MB 26kB/s \n",
            "\u001b[?25hRequirement already up-to-date: ktrain in /usr/local/lib/python3.6/dist-packages (0.14.2)\n",
            "Requirement already satisfied, skipping upgrade: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from pandas) (1.18.3)\n",
            "Requirement already satisfied, skipping upgrade: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas) (2018.9)\n",
            "Requirement already satisfied, skipping upgrade: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from pandas) (2.8.1)\n",
            "Requirement already satisfied, skipping upgrade: opt-einsum>=2.3.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow_gpu) (3.2.1)\n",
            "Requirement already satisfied, skipping upgrade: gast==0.2.2 in /usr/local/lib/python3.6/dist-packages (from tensorflow_gpu) (0.2.2)\n",
            "Requirement already satisfied, skipping upgrade: tensorflow-estimator<2.2.0,>=2.1.0rc0 in /usr/local/lib/python3.6/dist-packages (from tensorflow_gpu) (2.1.0)\n",
            "Requirement already satisfied, skipping upgrade: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow_gpu) (1.1.0)\n",
            "Requirement already satisfied, skipping upgrade: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow_gpu) (1.12.1)\n",
            "Requirement already satisfied, skipping upgrade: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow_gpu) (1.28.1)\n",
            "Requirement already satisfied, skipping upgrade: scipy==1.4.1; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tensorflow_gpu) (1.4.1)\n",
            "Requirement already satisfied, skipping upgrade: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow_gpu) (0.8.1)\n",
            "Requirement already satisfied, skipping upgrade: keras-preprocessing>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow_gpu) (1.1.0)\n",
            "Requirement already satisfied, skipping upgrade: protobuf>=3.8.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow_gpu) (3.10.0)\n",
            "Requirement already satisfied, skipping upgrade: tensorboard<2.2.0,>=2.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow_gpu) (2.1.1)\n",
            "Requirement already satisfied, skipping upgrade: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow_gpu) (0.9.0)\n",
            "Requirement already satisfied, skipping upgrade: keras-applications>=1.0.8 in /usr/local/lib/python3.6/dist-packages (from tensorflow_gpu) (1.0.8)\n",
            "Requirement already satisfied, skipping upgrade: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow_gpu) (0.2.0)\n",
            "Requirement already satisfied, skipping upgrade: wheel>=0.26; python_version >= \"3\" in /usr/local/lib/python3.6/dist-packages (from tensorflow_gpu) (0.34.2)\n",
            "Requirement already satisfied, skipping upgrade: six>=1.12.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow_gpu) (1.12.0)\n",
            "Requirement already satisfied, skipping upgrade: whoosh in /usr/local/lib/python3.6/dist-packages (from ktrain) (2.7.4)\n",
            "Requirement already satisfied, skipping upgrade: syntok in /usr/local/lib/python3.6/dist-packages (from ktrain) (1.2.2)\n",
            "Requirement already satisfied, skipping upgrade: tensorflow-datasets in /usr/local/lib/python3.6/dist-packages (from ktrain) (2.1.0)\n",
            "Requirement already satisfied, skipping upgrade: requests in /usr/local/lib/python3.6/dist-packages (from ktrain) (2.21.0)\n",
            "Requirement already satisfied, skipping upgrade: matplotlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from ktrain) (3.2.1)\n",
            "Requirement already satisfied, skipping upgrade: scikit-learn>=0.21.3 in /usr/local/lib/python3.6/dist-packages (from ktrain) (0.22.2.post1)\n",
            "Requirement already satisfied, skipping upgrade: langdetect in /usr/local/lib/python3.6/dist-packages (from ktrain) (1.0.8)\n",
            "Requirement already satisfied, skipping upgrade: packaging in /usr/local/lib/python3.6/dist-packages (from ktrain) (20.3)\n",
            "Requirement already satisfied, skipping upgrade: bokeh in /usr/local/lib/python3.6/dist-packages (from ktrain) (1.4.0)\n",
            "Requirement already satisfied, skipping upgrade: fastprogress>=0.1.21 in /usr/local/lib/python3.6/dist-packages (from ktrain) (0.2.3)\n",
            "Requirement already satisfied, skipping upgrade: transformers>=2.7.0 in /usr/local/lib/python3.6/dist-packages (from ktrain) (2.8.0)\n",
            "Requirement already satisfied, skipping upgrade: ipython in /usr/local/lib/python3.6/dist-packages (from ktrain) (5.5.0)\n",
            "Requirement already satisfied, skipping upgrade: keras-bert>=0.81.0 in /usr/local/lib/python3.6/dist-packages (from ktrain) (0.81.0)\n",
            "Requirement already satisfied, skipping upgrade: tensorflow==2.1.0 in /usr/local/lib/python3.6/dist-packages (from ktrain) (2.1.0)\n",
            "Requirement already satisfied, skipping upgrade: joblib in /usr/local/lib/python3.6/dist-packages (from ktrain) (0.14.1)\n",
            "Requirement already satisfied, skipping upgrade: jieba in /usr/local/lib/python3.6/dist-packages (from ktrain) (0.42.1)\n",
            "Requirement already satisfied, skipping upgrade: cchardet in /usr/local/lib/python3.6/dist-packages (from ktrain) (2.1.6)\n",
            "Requirement already satisfied, skipping upgrade: seqeval in /usr/local/lib/python3.6/dist-packages (from ktrain) (0.0.12)\n",
            "Requirement already satisfied, skipping upgrade: networkx>=2.3 in /usr/local/lib/python3.6/dist-packages (from ktrain) (2.4)\n",
            "Requirement already satisfied, skipping upgrade: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.8.0->tensorflow_gpu) (46.1.3)\n",
            "Requirement already satisfied, skipping upgrade: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow_gpu) (0.4.1)\n",
            "Requirement already satisfied, skipping upgrade: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow_gpu) (1.0.1)\n",
            "Requirement already satisfied, skipping upgrade: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow_gpu) (3.2.1)\n",
            "Requirement already satisfied, skipping upgrade: google-auth<2,>=1.6.3 in /usr/local/lib/python3.6/dist-packages (from tensorboard<2.2.0,>=2.1.0->tensorflow_gpu) (1.7.2)\n",
            "Requirement already satisfied, skipping upgrade: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.8->tensorflow_gpu) (2.10.0)\n",
            "Requirement already satisfied, skipping upgrade: regex in /usr/local/lib/python3.6/dist-packages (from syntok->ktrain) (2019.12.20)\n",
            "Requirement already satisfied, skipping upgrade: tqdm in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->ktrain) (4.38.0)\n",
            "Requirement already satisfied, skipping upgrade: promise in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->ktrain) (2.3)\n",
            "Requirement already satisfied, skipping upgrade: attrs>=18.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->ktrain) (19.3.0)\n",
            "Requirement already satisfied, skipping upgrade: dill in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->ktrain) (0.3.1.1)\n",
            "Requirement already satisfied, skipping upgrade: future in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->ktrain) (0.16.0)\n",
            "Requirement already satisfied, skipping upgrade: tensorflow-metadata in /usr/local/lib/python3.6/dist-packages (from tensorflow-datasets->ktrain) (0.21.2)\n",
            "Requirement already satisfied, skipping upgrade: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->ktrain) (1.24.3)\n",
            "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->ktrain) (2020.4.5.1)\n",
            "Requirement already satisfied, skipping upgrade: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->ktrain) (2.8)\n",
            "Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->ktrain) (3.0.4)\n",
            "Requirement already satisfied, skipping upgrade: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0.0->ktrain) (2.4.7)\n",
            "Requirement already satisfied, skipping upgrade: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0.0->ktrain) (1.2.0)\n",
            "Requirement already satisfied, skipping upgrade: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=3.0.0->ktrain) (0.10.0)\n",
            "Requirement already satisfied, skipping upgrade: Jinja2>=2.7 in /usr/local/lib/python3.6/dist-packages (from bokeh->ktrain) (2.11.2)\n",
            "Requirement already satisfied, skipping upgrade: pillow>=4.0 in /usr/local/lib/python3.6/dist-packages (from bokeh->ktrain) (7.0.0)\n",
            "Requirement already satisfied, skipping upgrade: tornado>=4.3 in /usr/local/lib/python3.6/dist-packages (from bokeh->ktrain) (4.5.3)\n",
            "Requirement already satisfied, skipping upgrade: PyYAML>=3.10 in /usr/local/lib/python3.6/dist-packages (from bokeh->ktrain) (3.13)\n",
            "Requirement already satisfied, skipping upgrade: sentencepiece in /usr/local/lib/python3.6/dist-packages (from transformers>=2.7.0->ktrain) (0.1.86)\n",
            "Requirement already satisfied, skipping upgrade: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers>=2.7.0->ktrain) (0.0.41)\n",
            "Requirement already satisfied, skipping upgrade: filelock in /usr/local/lib/python3.6/dist-packages (from transformers>=2.7.0->ktrain) (3.0.12)\n",
            "Requirement already satisfied, skipping upgrade: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers>=2.7.0->ktrain) (1.12.43)\n",
            "Requirement already satisfied, skipping upgrade: tokenizers==0.5.2 in /usr/local/lib/python3.6/dist-packages (from transformers>=2.7.0->ktrain) (0.5.2)\n",
            "Requirement already satisfied, skipping upgrade: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers>=2.7.0->ktrain) (0.7)\n",
            "Requirement already satisfied, skipping upgrade: traitlets>=4.2 in /usr/local/lib/python3.6/dist-packages (from ipython->ktrain) (4.3.3)\n",
            "Requirement already satisfied, skipping upgrade: pickleshare in /usr/local/lib/python3.6/dist-packages (from ipython->ktrain) (0.7.5)\n",
            "Requirement already satisfied, skipping upgrade: pygments in /usr/local/lib/python3.6/dist-packages (from ipython->ktrain) (2.1.3)\n",
            "Requirement already satisfied, skipping upgrade: simplegeneric>0.8 in /usr/local/lib/python3.6/dist-packages (from ipython->ktrain) (0.8.1)\n",
            "Requirement already satisfied, skipping upgrade: pexpect; sys_platform != \"win32\" in /usr/local/lib/python3.6/dist-packages (from ipython->ktrain) (4.8.0)\n",
            "Requirement already satisfied, skipping upgrade: decorator in /usr/local/lib/python3.6/dist-packages (from ipython->ktrain) (4.4.2)\n",
            "Requirement already satisfied, skipping upgrade: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.6/dist-packages (from ipython->ktrain) (1.0.18)\n",
            "Requirement already satisfied, skipping upgrade: Keras in /usr/local/lib/python3.6/dist-packages (from keras-bert>=0.81.0->ktrain) (2.3.1)\n",
            "Requirement already satisfied, skipping upgrade: keras-transformer>=0.30.0 in /usr/local/lib/python3.6/dist-packages (from keras-bert>=0.81.0->ktrain) (0.32.0)\n",
            "Requirement already satisfied, skipping upgrade: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.2.0,>=2.1.0->tensorflow_gpu) (1.3.0)\n",
            "Requirement already satisfied, skipping upgrade: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow_gpu) (0.2.8)\n",
            "Requirement already satisfied, skipping upgrade: rsa<4.1,>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow_gpu) (4.0)\n",
            "Requirement already satisfied, skipping upgrade: cachetools<3.2,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow_gpu) (3.1.1)\n",
            "Requirement already satisfied, skipping upgrade: googleapis-common-protos in /usr/local/lib/python3.6/dist-packages (from tensorflow-metadata->tensorflow-datasets->ktrain) (1.51.0)\n",
            "Requirement already satisfied, skipping upgrade: MarkupSafe>=0.23 in /usr/local/lib/python3.6/dist-packages (from Jinja2>=2.7->bokeh->ktrain) (1.1.1)\n",
            "Requirement already satisfied, skipping upgrade: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers>=2.7.0->ktrain) (7.1.1)\n",
            "Requirement already satisfied, skipping upgrade: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.7.0->ktrain) (0.3.3)\n",
            "Requirement already satisfied, skipping upgrade: botocore<1.16.0,>=1.15.43 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.7.0->ktrain) (1.15.43)\n",
            "Requirement already satisfied, skipping upgrade: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.7.0->ktrain) (0.9.5)\n",
            "Requirement already satisfied, skipping upgrade: ipython-genutils in /usr/local/lib/python3.6/dist-packages (from traitlets>=4.2->ipython->ktrain) (0.2.0)\n",
            "Requirement already satisfied, skipping upgrade: ptyprocess>=0.5 in /usr/local/lib/python3.6/dist-packages (from pexpect; sys_platform != \"win32\"->ipython->ktrain) (0.6.0)\n",
            "Requirement already satisfied, skipping upgrade: wcwidth in /usr/local/lib/python3.6/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython->ktrain) (0.1.9)\n",
            "Requirement already satisfied, skipping upgrade: keras-position-wise-feed-forward>=0.5.0 in /usr/local/lib/python3.6/dist-packages (from keras-transformer>=0.30.0->keras-bert>=0.81.0->ktrain) (0.6.0)\n",
            "Requirement already satisfied, skipping upgrade: keras-layer-normalization>=0.12.0 in /usr/local/lib/python3.6/dist-packages (from keras-transformer>=0.30.0->keras-bert>=0.81.0->ktrain) (0.14.0)\n",
            "Requirement already satisfied, skipping upgrade: keras-embed-sim>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from keras-transformer>=0.30.0->keras-bert>=0.81.0->ktrain) (0.7.0)\n",
            "Requirement already satisfied, skipping upgrade: keras-multi-head>=0.22.0 in /usr/local/lib/python3.6/dist-packages (from keras-transformer>=0.30.0->keras-bert>=0.81.0->ktrain) (0.22.0)\n",
            "Requirement already satisfied, skipping upgrade: keras-pos-embd>=0.10.0 in /usr/local/lib/python3.6/dist-packages (from keras-transformer>=0.30.0->keras-bert>=0.81.0->ktrain) (0.11.0)\n",
            "Requirement already satisfied, skipping upgrade: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.2.0,>=2.1.0->tensorflow_gpu) (3.1.0)\n",
            "Requirement already satisfied, skipping upgrade: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.6/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard<2.2.0,>=2.1.0->tensorflow_gpu) (0.4.8)\n",
            "Requirement already satisfied, skipping upgrade: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.16.0,>=1.15.43->boto3->transformers>=2.7.0->ktrain) (0.15.2)\n",
            "Requirement already satisfied, skipping upgrade: keras-self-attention==0.41.0 in /usr/local/lib/python3.6/dist-packages (from keras-multi-head>=0.22.0->keras-transformer>=0.30.0->keras-bert>=0.81.0->ktrain) (0.41.0)\n",
            "Installing collected packages: tensorflow-gpu\n",
            "Successfully installed tensorflow-gpu-2.1.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "tensorflow",
                  "tensorflow_core"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S-m1NBtpL13h",
        "colab_type": "code",
        "outputId": "2b687cc3-3c2d-4907-f1e0-a8b279612f02",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 82
        }
      },
      "source": [
        "!nvcc --version"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "nvcc: NVIDIA (R) Cuda compiler driver\n",
            "Copyright (c) 2005-2019 NVIDIA Corporation\n",
            "Built on Sun_Jul_28_19:07:16_PDT_2019\n",
            "Cuda compilation tools, release 10.1, V10.1.243\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yOuRyPNh9Iyw",
        "colab_type": "code",
        "outputId": "58b59f11-4879-46a5-8a13-704ac1afb7a0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "! rm -rf apex\n",
        "! git clone https://www.github.com/nvidia/apex\n",
        "! cd apex && python setup.py install"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'apex'...\n",
            "warning: redirecting to https://github.com/nvidia/apex.git/\n",
            "remote: Enumerating objects: 4, done.\u001b[K\n",
            "remote: Counting objects: 100% (4/4), done.\u001b[K\n",
            "remote: Compressing objects: 100% (4/4), done.\u001b[K\n",
            "remote: Total 6593 (delta 0), reused 2 (delta 0), pack-reused 6589\u001b[K\n",
            "Receiving objects: 100% (6593/6593), 13.71 MiB | 13.53 MiB/s, done.\n",
            "Resolving deltas: 100% (4382/4382), done.\n",
            "torch.__version__  =  1.4.0\n",
            "setup.py:46: UserWarning: Option --pyprof not specified. Not installing PyProf dependencies!\n",
            "  warnings.warn(\"Option --pyprof not specified. Not installing PyProf dependencies!\")\n",
            "running install\n",
            "running bdist_egg\n",
            "running egg_info\n",
            "creating apex.egg-info\n",
            "writing apex.egg-info/PKG-INFO\n",
            "writing dependency_links to apex.egg-info/dependency_links.txt\n",
            "writing top-level names to apex.egg-info/top_level.txt\n",
            "writing manifest file 'apex.egg-info/SOURCES.txt'\n",
            "writing manifest file 'apex.egg-info/SOURCES.txt'\n",
            "installing library code to build/bdist.linux-x86_64/egg\n",
            "running install_lib\n",
            "running build_py\n",
            "creating build\n",
            "creating build/lib\n",
            "creating build/lib/apex\n",
            "copying apex/__init__.py -> build/lib/apex\n",
            "creating build/lib/apex/pyprof\n",
            "copying apex/pyprof/__init__.py -> build/lib/apex/pyprof\n",
            "creating build/lib/apex/contrib\n",
            "copying apex/contrib/__init__.py -> build/lib/apex/contrib\n",
            "creating build/lib/apex/normalization\n",
            "copying apex/normalization/fused_layer_norm.py -> build/lib/apex/normalization\n",
            "copying apex/normalization/__init__.py -> build/lib/apex/normalization\n",
            "creating build/lib/apex/mlp\n",
            "copying apex/mlp/mlp.py -> build/lib/apex/mlp\n",
            "copying apex/mlp/__init__.py -> build/lib/apex/mlp\n",
            "creating build/lib/apex/RNN\n",
            "copying apex/RNN/models.py -> build/lib/apex/RNN\n",
            "copying apex/RNN/RNNBackend.py -> build/lib/apex/RNN\n",
            "copying apex/RNN/__init__.py -> build/lib/apex/RNN\n",
            "copying apex/RNN/cells.py -> build/lib/apex/RNN\n",
            "creating build/lib/apex/multi_tensor_apply\n",
            "copying apex/multi_tensor_apply/multi_tensor_apply.py -> build/lib/apex/multi_tensor_apply\n",
            "copying apex/multi_tensor_apply/__init__.py -> build/lib/apex/multi_tensor_apply\n",
            "creating build/lib/apex/optimizers\n",
            "copying apex/optimizers/fused_adam.py -> build/lib/apex/optimizers\n",
            "copying apex/optimizers/fused_lamb.py -> build/lib/apex/optimizers\n",
            "copying apex/optimizers/fused_novograd.py -> build/lib/apex/optimizers\n",
            "copying apex/optimizers/__init__.py -> build/lib/apex/optimizers\n",
            "copying apex/optimizers/fused_sgd.py -> build/lib/apex/optimizers\n",
            "creating build/lib/apex/parallel\n",
            "copying apex/parallel/multiproc.py -> build/lib/apex/parallel\n",
            "copying apex/parallel/sync_batchnorm.py -> build/lib/apex/parallel\n",
            "copying apex/parallel/LARC.py -> build/lib/apex/parallel\n",
            "copying apex/parallel/distributed.py -> build/lib/apex/parallel\n",
            "copying apex/parallel/optimized_sync_batchnorm.py -> build/lib/apex/parallel\n",
            "copying apex/parallel/sync_batchnorm_kernel.py -> build/lib/apex/parallel\n",
            "copying apex/parallel/optimized_sync_batchnorm_kernel.py -> build/lib/apex/parallel\n",
            "copying apex/parallel/__init__.py -> build/lib/apex/parallel\n",
            "creating build/lib/apex/reparameterization\n",
            "copying apex/reparameterization/__init__.py -> build/lib/apex/reparameterization\n",
            "copying apex/reparameterization/reparameterization.py -> build/lib/apex/reparameterization\n",
            "copying apex/reparameterization/weight_norm.py -> build/lib/apex/reparameterization\n",
            "creating build/lib/apex/amp\n",
            "copying apex/amp/_process_optimizer.py -> build/lib/apex/amp\n",
            "copying apex/amp/__version__.py -> build/lib/apex/amp\n",
            "copying apex/amp/compat.py -> build/lib/apex/amp\n",
            "copying apex/amp/handle.py -> build/lib/apex/amp\n",
            "copying apex/amp/_initialize.py -> build/lib/apex/amp\n",
            "copying apex/amp/rnn_compat.py -> build/lib/apex/amp\n",
            "copying apex/amp/amp.py -> build/lib/apex/amp\n",
            "copying apex/amp/opt.py -> build/lib/apex/amp\n",
            "copying apex/amp/__init__.py -> build/lib/apex/amp\n",
            "copying apex/amp/utils.py -> build/lib/apex/amp\n",
            "copying apex/amp/frontend.py -> build/lib/apex/amp\n",
            "copying apex/amp/_amp_state.py -> build/lib/apex/amp\n",
            "copying apex/amp/wrap.py -> build/lib/apex/amp\n",
            "copying apex/amp/scaler.py -> build/lib/apex/amp\n",
            "creating build/lib/apex/fp16_utils\n",
            "copying apex/fp16_utils/fp16util.py -> build/lib/apex/fp16_utils\n",
            "copying apex/fp16_utils/fp16_optimizer.py -> build/lib/apex/fp16_utils\n",
            "copying apex/fp16_utils/__init__.py -> build/lib/apex/fp16_utils\n",
            "copying apex/fp16_utils/loss_scaler.py -> build/lib/apex/fp16_utils\n",
            "creating build/lib/apex/pyprof/nvtx\n",
            "copying apex/pyprof/nvtx/__init__.py -> build/lib/apex/pyprof/nvtx\n",
            "copying apex/pyprof/nvtx/nvmarker.py -> build/lib/apex/pyprof/nvtx\n",
            "creating build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/conv.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/blas.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/prof.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/softmax.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/recurrentCell.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/misc.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/base.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/embedding.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/usage.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/activation.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/loss.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/optim.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/dropout.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/pointwise.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/data.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/reduction.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/output.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/index_slice_join_mutate.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/pooling.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/__main__.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/__init__.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/linear.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/utility.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/randomSample.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/convert.py -> build/lib/apex/pyprof/prof\n",
            "copying apex/pyprof/prof/normalization.py -> build/lib/apex/pyprof/prof\n",
            "creating build/lib/apex/pyprof/parse\n",
            "copying apex/pyprof/parse/kernel.py -> build/lib/apex/pyprof/parse\n",
            "copying apex/pyprof/parse/nvvp.py -> build/lib/apex/pyprof/parse\n",
            "copying apex/pyprof/parse/parse.py -> build/lib/apex/pyprof/parse\n",
            "copying apex/pyprof/parse/__main__.py -> build/lib/apex/pyprof/parse\n",
            "copying apex/pyprof/parse/__init__.py -> build/lib/apex/pyprof/parse\n",
            "copying apex/pyprof/parse/db.py -> build/lib/apex/pyprof/parse\n",
            "creating build/lib/apex/contrib/groupbn\n",
            "copying apex/contrib/groupbn/batch_norm.py -> build/lib/apex/contrib/groupbn\n",
            "copying apex/contrib/groupbn/__init__.py -> build/lib/apex/contrib/groupbn\n",
            "creating build/lib/apex/contrib/xentropy\n",
            "copying apex/contrib/xentropy/__init__.py -> build/lib/apex/contrib/xentropy\n",
            "copying apex/contrib/xentropy/softmax_xentropy.py -> build/lib/apex/contrib/xentropy\n",
            "creating build/lib/apex/contrib/optimizers\n",
            "copying apex/contrib/optimizers/fused_adam.py -> build/lib/apex/contrib/optimizers\n",
            "copying apex/contrib/optimizers/fp16_optimizer.py -> build/lib/apex/contrib/optimizers\n",
            "copying apex/contrib/optimizers/fused_lamb.py -> build/lib/apex/contrib/optimizers\n",
            "copying apex/contrib/optimizers/__init__.py -> build/lib/apex/contrib/optimizers\n",
            "copying apex/contrib/optimizers/fused_sgd.py -> build/lib/apex/contrib/optimizers\n",
            "creating build/lib/apex/contrib/multihead_attn\n",
            "copying apex/contrib/multihead_attn/self_multihead_attn_func.py -> build/lib/apex/contrib/multihead_attn\n",
            "copying apex/contrib/multihead_attn/encdec_multihead_attn.py -> build/lib/apex/contrib/multihead_attn\n",
            "copying apex/contrib/multihead_attn/fast_encdec_multihead_attn_func.py -> build/lib/apex/contrib/multihead_attn\n",
            "copying apex/contrib/multihead_attn/fast_self_multihead_attn_norm_add_func.py -> build/lib/apex/contrib/multihead_attn\n",
            "copying apex/contrib/multihead_attn/self_multihead_attn.py -> build/lib/apex/contrib/multihead_attn\n",
            "copying apex/contrib/multihead_attn/fast_self_multihead_attn_func.py -> build/lib/apex/contrib/multihead_attn\n",
            "copying apex/contrib/multihead_attn/__init__.py -> build/lib/apex/contrib/multihead_attn\n",
            "copying apex/contrib/multihead_attn/encdec_multihead_attn_func.py -> build/lib/apex/contrib/multihead_attn\n",
            "copying apex/contrib/multihead_attn/fast_encdec_multihead_attn_norm_add_func.py -> build/lib/apex/contrib/multihead_attn\n",
            "creating build/lib/apex/amp/lists\n",
            "copying apex/amp/lists/tensor_overrides.py -> build/lib/apex/amp/lists\n",
            "copying apex/amp/lists/functional_overrides.py -> build/lib/apex/amp/lists\n",
            "copying apex/amp/lists/torch_overrides.py -> build/lib/apex/amp/lists\n",
            "copying apex/amp/lists/__init__.py -> build/lib/apex/amp/lists\n",
            "creating build/bdist.linux-x86_64\n",
            "creating build/bdist.linux-x86_64/egg\n",
            "creating build/bdist.linux-x86_64/egg/apex\n",
            "creating build/bdist.linux-x86_64/egg/apex/pyprof\n",
            "creating build/bdist.linux-x86_64/egg/apex/pyprof/nvtx\n",
            "copying build/lib/apex/pyprof/nvtx/__init__.py -> build/bdist.linux-x86_64/egg/apex/pyprof/nvtx\n",
            "copying build/lib/apex/pyprof/nvtx/nvmarker.py -> build/bdist.linux-x86_64/egg/apex/pyprof/nvtx\n",
            "creating build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/conv.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/blas.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/prof.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/softmax.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/recurrentCell.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/misc.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/base.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/embedding.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/usage.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/activation.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/loss.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/optim.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/dropout.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/pointwise.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/data.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/reduction.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/output.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/index_slice_join_mutate.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/pooling.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/__main__.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/__init__.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/linear.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/utility.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/randomSample.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/convert.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/prof/normalization.py -> build/bdist.linux-x86_64/egg/apex/pyprof/prof\n",
            "copying build/lib/apex/pyprof/__init__.py -> build/bdist.linux-x86_64/egg/apex/pyprof\n",
            "creating build/bdist.linux-x86_64/egg/apex/pyprof/parse\n",
            "copying build/lib/apex/pyprof/parse/kernel.py -> build/bdist.linux-x86_64/egg/apex/pyprof/parse\n",
            "copying build/lib/apex/pyprof/parse/nvvp.py -> build/bdist.linux-x86_64/egg/apex/pyprof/parse\n",
            "copying build/lib/apex/pyprof/parse/parse.py -> build/bdist.linux-x86_64/egg/apex/pyprof/parse\n",
            "copying build/lib/apex/pyprof/parse/__main__.py -> build/bdist.linux-x86_64/egg/apex/pyprof/parse\n",
            "copying build/lib/apex/pyprof/parse/__init__.py -> build/bdist.linux-x86_64/egg/apex/pyprof/parse\n",
            "copying build/lib/apex/pyprof/parse/db.py -> build/bdist.linux-x86_64/egg/apex/pyprof/parse\n",
            "creating build/bdist.linux-x86_64/egg/apex/contrib\n",
            "creating build/bdist.linux-x86_64/egg/apex/contrib/groupbn\n",
            "copying build/lib/apex/contrib/groupbn/batch_norm.py -> build/bdist.linux-x86_64/egg/apex/contrib/groupbn\n",
            "copying build/lib/apex/contrib/groupbn/__init__.py -> build/bdist.linux-x86_64/egg/apex/contrib/groupbn\n",
            "creating build/bdist.linux-x86_64/egg/apex/contrib/xentropy\n",
            "copying build/lib/apex/contrib/xentropy/__init__.py -> build/bdist.linux-x86_64/egg/apex/contrib/xentropy\n",
            "copying build/lib/apex/contrib/xentropy/softmax_xentropy.py -> build/bdist.linux-x86_64/egg/apex/contrib/xentropy\n",
            "creating build/bdist.linux-x86_64/egg/apex/contrib/optimizers\n",
            "copying build/lib/apex/contrib/optimizers/fused_adam.py -> build/bdist.linux-x86_64/egg/apex/contrib/optimizers\n",
            "copying build/lib/apex/contrib/optimizers/fp16_optimizer.py -> build/bdist.linux-x86_64/egg/apex/contrib/optimizers\n",
            "copying build/lib/apex/contrib/optimizers/fused_lamb.py -> build/bdist.linux-x86_64/egg/apex/contrib/optimizers\n",
            "copying build/lib/apex/contrib/optimizers/__init__.py -> build/bdist.linux-x86_64/egg/apex/contrib/optimizers\n",
            "copying build/lib/apex/contrib/optimizers/fused_sgd.py -> build/bdist.linux-x86_64/egg/apex/contrib/optimizers\n",
            "creating build/bdist.linux-x86_64/egg/apex/contrib/multihead_attn\n",
            "copying build/lib/apex/contrib/multihead_attn/self_multihead_attn_func.py -> build/bdist.linux-x86_64/egg/apex/contrib/multihead_attn\n",
            "copying build/lib/apex/contrib/multihead_attn/encdec_multihead_attn.py -> build/bdist.linux-x86_64/egg/apex/contrib/multihead_attn\n",
            "copying build/lib/apex/contrib/multihead_attn/fast_encdec_multihead_attn_func.py -> build/bdist.linux-x86_64/egg/apex/contrib/multihead_attn\n",
            "copying build/lib/apex/contrib/multihead_attn/fast_self_multihead_attn_norm_add_func.py -> build/bdist.linux-x86_64/egg/apex/contrib/multihead_attn\n",
            "copying build/lib/apex/contrib/multihead_attn/self_multihead_attn.py -> build/bdist.linux-x86_64/egg/apex/contrib/multihead_attn\n",
            "copying build/lib/apex/contrib/multihead_attn/fast_self_multihead_attn_func.py -> build/bdist.linux-x86_64/egg/apex/contrib/multihead_attn\n",
            "copying build/lib/apex/contrib/multihead_attn/__init__.py -> build/bdist.linux-x86_64/egg/apex/contrib/multihead_attn\n",
            "copying build/lib/apex/contrib/multihead_attn/encdec_multihead_attn_func.py -> build/bdist.linux-x86_64/egg/apex/contrib/multihead_attn\n",
            "copying build/lib/apex/contrib/multihead_attn/fast_encdec_multihead_attn_norm_add_func.py -> build/bdist.linux-x86_64/egg/apex/contrib/multihead_attn\n",
            "copying build/lib/apex/contrib/__init__.py -> build/bdist.linux-x86_64/egg/apex/contrib\n",
            "creating build/bdist.linux-x86_64/egg/apex/normalization\n",
            "copying build/lib/apex/normalization/fused_layer_norm.py -> build/bdist.linux-x86_64/egg/apex/normalization\n",
            "copying build/lib/apex/normalization/__init__.py -> build/bdist.linux-x86_64/egg/apex/normalization\n",
            "creating build/bdist.linux-x86_64/egg/apex/mlp\n",
            "copying build/lib/apex/mlp/mlp.py -> build/bdist.linux-x86_64/egg/apex/mlp\n",
            "copying build/lib/apex/mlp/__init__.py -> build/bdist.linux-x86_64/egg/apex/mlp\n",
            "creating build/bdist.linux-x86_64/egg/apex/RNN\n",
            "copying build/lib/apex/RNN/models.py -> build/bdist.linux-x86_64/egg/apex/RNN\n",
            "copying build/lib/apex/RNN/RNNBackend.py -> build/bdist.linux-x86_64/egg/apex/RNN\n",
            "copying build/lib/apex/RNN/__init__.py -> build/bdist.linux-x86_64/egg/apex/RNN\n",
            "copying build/lib/apex/RNN/cells.py -> build/bdist.linux-x86_64/egg/apex/RNN\n",
            "creating build/bdist.linux-x86_64/egg/apex/multi_tensor_apply\n",
            "copying build/lib/apex/multi_tensor_apply/multi_tensor_apply.py -> build/bdist.linux-x86_64/egg/apex/multi_tensor_apply\n",
            "copying build/lib/apex/multi_tensor_apply/__init__.py -> build/bdist.linux-x86_64/egg/apex/multi_tensor_apply\n",
            "creating build/bdist.linux-x86_64/egg/apex/optimizers\n",
            "copying build/lib/apex/optimizers/fused_adam.py -> build/bdist.linux-x86_64/egg/apex/optimizers\n",
            "copying build/lib/apex/optimizers/fused_lamb.py -> build/bdist.linux-x86_64/egg/apex/optimizers\n",
            "copying build/lib/apex/optimizers/fused_novograd.py -> build/bdist.linux-x86_64/egg/apex/optimizers\n",
            "copying build/lib/apex/optimizers/__init__.py -> build/bdist.linux-x86_64/egg/apex/optimizers\n",
            "copying build/lib/apex/optimizers/fused_sgd.py -> build/bdist.linux-x86_64/egg/apex/optimizers\n",
            "creating build/bdist.linux-x86_64/egg/apex/parallel\n",
            "copying build/lib/apex/parallel/multiproc.py -> build/bdist.linux-x86_64/egg/apex/parallel\n",
            "copying build/lib/apex/parallel/sync_batchnorm.py -> build/bdist.linux-x86_64/egg/apex/parallel\n",
            "copying build/lib/apex/parallel/LARC.py -> build/bdist.linux-x86_64/egg/apex/parallel\n",
            "copying build/lib/apex/parallel/distributed.py -> build/bdist.linux-x86_64/egg/apex/parallel\n",
            "copying build/lib/apex/parallel/optimized_sync_batchnorm.py -> build/bdist.linux-x86_64/egg/apex/parallel\n",
            "copying build/lib/apex/parallel/sync_batchnorm_kernel.py -> build/bdist.linux-x86_64/egg/apex/parallel\n",
            "copying build/lib/apex/parallel/optimized_sync_batchnorm_kernel.py -> build/bdist.linux-x86_64/egg/apex/parallel\n",
            "copying build/lib/apex/parallel/__init__.py -> build/bdist.linux-x86_64/egg/apex/parallel\n",
            "creating build/bdist.linux-x86_64/egg/apex/reparameterization\n",
            "copying build/lib/apex/reparameterization/__init__.py -> build/bdist.linux-x86_64/egg/apex/reparameterization\n",
            "copying build/lib/apex/reparameterization/reparameterization.py -> build/bdist.linux-x86_64/egg/apex/reparameterization\n",
            "copying build/lib/apex/reparameterization/weight_norm.py -> build/bdist.linux-x86_64/egg/apex/reparameterization\n",
            "creating build/bdist.linux-x86_64/egg/apex/amp\n",
            "copying build/lib/apex/amp/_process_optimizer.py -> build/bdist.linux-x86_64/egg/apex/amp\n",
            "copying build/lib/apex/amp/__version__.py -> build/bdist.linux-x86_64/egg/apex/amp\n",
            "copying build/lib/apex/amp/compat.py -> build/bdist.linux-x86_64/egg/apex/amp\n",
            "copying build/lib/apex/amp/handle.py -> build/bdist.linux-x86_64/egg/apex/amp\n",
            "copying build/lib/apex/amp/_initialize.py -> build/bdist.linux-x86_64/egg/apex/amp\n",
            "copying build/lib/apex/amp/rnn_compat.py -> build/bdist.linux-x86_64/egg/apex/amp\n",
            "copying build/lib/apex/amp/amp.py -> build/bdist.linux-x86_64/egg/apex/amp\n",
            "copying build/lib/apex/amp/opt.py -> build/bdist.linux-x86_64/egg/apex/amp\n",
            "copying build/lib/apex/amp/__init__.py -> build/bdist.linux-x86_64/egg/apex/amp\n",
            "copying build/lib/apex/amp/utils.py -> build/bdist.linux-x86_64/egg/apex/amp\n",
            "copying build/lib/apex/amp/frontend.py -> build/bdist.linux-x86_64/egg/apex/amp\n",
            "creating build/bdist.linux-x86_64/egg/apex/amp/lists\n",
            "copying build/lib/apex/amp/lists/tensor_overrides.py -> build/bdist.linux-x86_64/egg/apex/amp/lists\n",
            "copying build/lib/apex/amp/lists/functional_overrides.py -> build/bdist.linux-x86_64/egg/apex/amp/lists\n",
            "copying build/lib/apex/amp/lists/torch_overrides.py -> build/bdist.linux-x86_64/egg/apex/amp/lists\n",
            "copying build/lib/apex/amp/lists/__init__.py -> build/bdist.linux-x86_64/egg/apex/amp/lists\n",
            "copying build/lib/apex/amp/_amp_state.py -> build/bdist.linux-x86_64/egg/apex/amp\n",
            "copying build/lib/apex/amp/wrap.py -> build/bdist.linux-x86_64/egg/apex/amp\n",
            "copying build/lib/apex/amp/scaler.py -> build/bdist.linux-x86_64/egg/apex/amp\n",
            "copying build/lib/apex/__init__.py -> build/bdist.linux-x86_64/egg/apex\n",
            "creating build/bdist.linux-x86_64/egg/apex/fp16_utils\n",
            "copying build/lib/apex/fp16_utils/fp16util.py -> build/bdist.linux-x86_64/egg/apex/fp16_utils\n",
            "copying build/lib/apex/fp16_utils/fp16_optimizer.py -> build/bdist.linux-x86_64/egg/apex/fp16_utils\n",
            "copying build/lib/apex/fp16_utils/__init__.py -> build/bdist.linux-x86_64/egg/apex/fp16_utils\n",
            "copying build/lib/apex/fp16_utils/loss_scaler.py -> build/bdist.linux-x86_64/egg/apex/fp16_utils\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/nvtx/__init__.py to __init__.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/nvtx/nvmarker.py to nvmarker.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/conv.py to conv.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/blas.py to blas.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/prof.py to prof.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/softmax.py to softmax.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/recurrentCell.py to recurrentCell.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/misc.py to misc.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/base.py to base.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/embedding.py to embedding.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/usage.py to usage.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/activation.py to activation.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/loss.py to loss.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/optim.py to optim.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/dropout.py to dropout.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/pointwise.py to pointwise.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/data.py to data.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/reduction.py to reduction.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/output.py to output.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/index_slice_join_mutate.py to index_slice_join_mutate.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/pooling.py to pooling.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/__main__.py to __main__.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/__init__.py to __init__.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/linear.py to linear.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/utility.py to utility.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/randomSample.py to randomSample.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/convert.py to convert.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/prof/normalization.py to normalization.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/__init__.py to __init__.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/parse/kernel.py to kernel.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/parse/nvvp.py to nvvp.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/parse/parse.py to parse.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/parse/__main__.py to __main__.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/parse/__init__.py to __init__.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/pyprof/parse/db.py to db.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/contrib/groupbn/batch_norm.py to batch_norm.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/contrib/groupbn/__init__.py to __init__.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/contrib/xentropy/__init__.py to __init__.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/contrib/xentropy/softmax_xentropy.py to softmax_xentropy.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/contrib/optimizers/fused_adam.py to fused_adam.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/contrib/optimizers/fp16_optimizer.py to fp16_optimizer.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/contrib/optimizers/fused_lamb.py to fused_lamb.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/contrib/optimizers/__init__.py to __init__.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/contrib/optimizers/fused_sgd.py to fused_sgd.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/contrib/multihead_attn/self_multihead_attn_func.py to self_multihead_attn_func.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/contrib/multihead_attn/encdec_multihead_attn.py to encdec_multihead_attn.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/contrib/multihead_attn/fast_encdec_multihead_attn_func.py to fast_encdec_multihead_attn_func.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/contrib/multihead_attn/fast_self_multihead_attn_norm_add_func.py to fast_self_multihead_attn_norm_add_func.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/contrib/multihead_attn/self_multihead_attn.py to self_multihead_attn.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/contrib/multihead_attn/fast_self_multihead_attn_func.py to fast_self_multihead_attn_func.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/contrib/multihead_attn/__init__.py to __init__.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/contrib/multihead_attn/encdec_multihead_attn_func.py to encdec_multihead_attn_func.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/contrib/multihead_attn/fast_encdec_multihead_attn_norm_add_func.py to fast_encdec_multihead_attn_norm_add_func.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/contrib/__init__.py to __init__.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/normalization/fused_layer_norm.py to fused_layer_norm.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/normalization/__init__.py to __init__.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/mlp/mlp.py to mlp.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/mlp/__init__.py to __init__.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/RNN/models.py to models.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/RNN/RNNBackend.py to RNNBackend.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/RNN/__init__.py to __init__.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/RNN/cells.py to cells.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/multi_tensor_apply/multi_tensor_apply.py to multi_tensor_apply.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/multi_tensor_apply/__init__.py to __init__.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/optimizers/fused_adam.py to fused_adam.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/optimizers/fused_lamb.py to fused_lamb.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/optimizers/fused_novograd.py to fused_novograd.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/optimizers/__init__.py to __init__.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/optimizers/fused_sgd.py to fused_sgd.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/parallel/multiproc.py to multiproc.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/parallel/sync_batchnorm.py to sync_batchnorm.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/parallel/LARC.py to LARC.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/parallel/distributed.py to distributed.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/parallel/optimized_sync_batchnorm.py to optimized_sync_batchnorm.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/parallel/sync_batchnorm_kernel.py to sync_batchnorm_kernel.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/parallel/optimized_sync_batchnorm_kernel.py to optimized_sync_batchnorm_kernel.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/parallel/__init__.py to __init__.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/reparameterization/__init__.py to __init__.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/reparameterization/reparameterization.py to reparameterization.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/reparameterization/weight_norm.py to weight_norm.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/_process_optimizer.py to _process_optimizer.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/__version__.py to __version__.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/compat.py to compat.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/handle.py to handle.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/_initialize.py to _initialize.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/rnn_compat.py to rnn_compat.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/amp.py to amp.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/opt.py to opt.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/__init__.py to __init__.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/utils.py to utils.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/frontend.py to frontend.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/lists/tensor_overrides.py to tensor_overrides.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/lists/functional_overrides.py to functional_overrides.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/lists/torch_overrides.py to torch_overrides.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/lists/__init__.py to __init__.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/_amp_state.py to _amp_state.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/wrap.py to wrap.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/amp/scaler.py to scaler.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/__init__.py to __init__.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/fp16_utils/fp16util.py to fp16util.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/fp16_utils/fp16_optimizer.py to fp16_optimizer.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/fp16_utils/__init__.py to __init__.cpython-36.pyc\n",
            "byte-compiling build/bdist.linux-x86_64/egg/apex/fp16_utils/loss_scaler.py to loss_scaler.cpython-36.pyc\n",
            "creating build/bdist.linux-x86_64/egg/EGG-INFO\n",
            "copying apex.egg-info/PKG-INFO -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
            "copying apex.egg-info/SOURCES.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
            "copying apex.egg-info/dependency_links.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
            "copying apex.egg-info/top_level.txt -> build/bdist.linux-x86_64/egg/EGG-INFO\n",
            "zip_safe flag not set; analyzing archive contents...\n",
            "apex.pyprof.nvtx.__pycache__.nvmarker.cpython-36: module references __file__\n",
            "apex.pyprof.nvtx.__pycache__.nvmarker.cpython-36: module references __path__\n",
            "creating dist\n",
            "creating 'dist/apex-0.1-py3.6.egg' and adding 'build/bdist.linux-x86_64/egg' to it\n",
            "removing 'build/bdist.linux-x86_64/egg' (and everything under it)\n",
            "Processing apex-0.1-py3.6.egg\n",
            "creating /usr/local/lib/python3.6/dist-packages/apex-0.1-py3.6.egg\n",
            "Extracting apex-0.1-py3.6.egg to /usr/local/lib/python3.6/dist-packages\n",
            "Adding apex 0.1 to easy-install.pth file\n",
            "\n",
            "Installed /usr/local/lib/python3.6/dist-packages/apex-0.1-py3.6.egg\n",
            "Processing dependencies for apex==0.1\n",
            "Finished processing dependencies for apex==0.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r0fKNCGVbQu1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "! cp ./drive/My\\ Drive/mlin/hindicorpus/hi_3500.csv ./"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OBxAA3FNL_2c",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 115
        },
        "outputId": "3e76bbd0-a37b-48c7-fbe4-152b6f811aaa"
      },
      "source": [
        "import pandas as pd \n",
        "\n",
        "df = pd.read_csv(\"./hi_3500.csv\", header=0, names=['review', 'sentiment'])\n",
        "print(df.head())"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "                                              review sentiment\n",
            "0  गुमनाम है वतन पर मिटने वाले लोग आतन्कवादियों स...  negative\n",
            "1  ज़ंजीर बदली जा रही थी मैं समझा था रिहाई हो गयी है  negative\n",
            "2  यूपी में बड़े स्तर पर दंगे करवा सकती है बीजेपी...  negative\n",
            "3  अंग्रेजी नहीं आती है इसलिए हिन्दी ट्विट ज्यादा...  negative\n",
            "4                    कश्मीर में हो रहा है जल जिहाद ।  negative\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l5XutXILb53g",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "X_train, X_test, y_train, y_test = train_test_split(df['review'], df['sentiment'], random_state=91)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mmllbmrlYaHM",
        "colab_type": "code",
        "outputId": "099178a9-a79e-4484-a09f-6131bc605791",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 33
        }
      },
      "source": [
        "print(set(y_train.values))"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'neutral', 'positive', 'negative'}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kfBouPQ-D7ve",
        "colab_type": "text"
      },
      "source": [
        "ktrain code below based on https://towardsdatascience.com/text-classification-with-hugging-face-transformers-in-tensorflow-2-without-tears-ee50e4f3e7ed"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T8UXEFknQ8j4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import ktrain\n",
        "from ktrain import text\n",
        "t = text.Transformer(\"monsoon-nlp/hindi-bert\", maxlen=500, class_names=list(set(y_train.values)))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LpHDOs4RAglZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 167
        },
        "outputId": "e43fa4d5-62d1-44d3-b73c-b03f1b4ae5c7"
      },
      "source": [
        "trn = t.preprocess_train(X_train.to_numpy(), y_train.to_numpy())"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "preprocessing train...\n",
            "language: hi\n",
            "train sequence lengths:\n",
            "\tmean : 16\n",
            "\t95percentile : 27\n",
            "\t99percentile : 30\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ktrain/text/preprocessor.py:478: UserWarning: class_names argument was ignored, as they were extracted from string labels in dataset\n",
            "  if self.get_classes(): warnings.warn('class_names argument was ignored, as they were extracted from string labels in dataset')\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZV89CAEeCJbM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 115
        },
        "outputId": "2de215d0-5a1e-46dd-8d46-e4a6ddb81c56"
      },
      "source": [
        "evalr = t.preprocess_test(X_test.to_numpy(), y_test.to_numpy())"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "preprocessing test...\n",
            "language: hi\n",
            "test sequence lengths:\n",
            "\tmean : 16\n",
            "\t95percentile : 27\n",
            "\t99percentile : 30\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w-t5q1lqCPQr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model = t.get_classifier()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1uyC0FwWCioS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "learner = ktrain.get_learner(model, train_data=trn, val_data=evalr, batch_size=6)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "574A3qlwCo03",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 465
        },
        "outputId": "531768b9-3a55-4401-e157-04b788447374"
      },
      "source": [
        "learner.lr_find(show_plot=True, max_epochs=3)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "simulating training for different learning rates... this may take a few moments...\n",
            "Train for 1134 steps\n",
            "Epoch 1/3\n",
            "1134/1134 [==============================] - 166s 146ms/step - loss: 1.0288 - accuracy: 0.3967\n",
            "Epoch 2/3\n",
            "1134/1134 [==============================] - 149s 131ms/step - loss: 1.0132 - accuracy: 0.4126\n",
            "Epoch 3/3\n",
            "  63/1134 [>.............................] - ETA: 2:44 - loss: 2.9390 - accuracy: 0.3440\n",
            "\n",
            "done.\n",
            "Visually inspect the loss plot to help identify the maximal learning rate\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAENCAYAAAAVPvJNAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXxU9b3/8dcnmSyEQFgSBAkRUEQpsmhEraLYVovaVtvi2uu1rcrVLrftzy52UXu1rV3trVZrsVXb3qrXVu3VVsVd3FABWQQEWVRWCVtIINvMfH5/zEkcwiQEyJnJJO/n4zEPZ77ne875fBOZT77ne873a+6OiIhIazmZDkBERLomJQgREUlJCUJERFJSghARkZSUIEREJCUlCBERSSm0BGFmw8zsWTNbYmaLzexrKepMMbNqM5sfvK5N2jbVzJaZ2QozuzqsOEVEJLVIiMeOAle5+zwz6wPMNbMn3X1Jq3ovuPsnkgvMLBe4FTgNWAu8bmYPp9hXRERCEloPwt03uPu84H0NsBQY2sHdJwEr3H2VuzcC9wFnhxOpiIikEmYPooWZDQcmAq+m2HyCmS0A1gPfdPfFJBLJmqQ6a4Hj9nae0tJSHz58+IGGKyLSY8ydO3ezu5el2hZ6gjCzYuAB4OvuvqPV5nnAIe5ea2ZnAv8ARu3j8acD0wEqKiqYM2dOJ0QtItIzmNm7bW0L9S4mM8sjkRz+6u4Ptt7u7jvcvTZ4/yiQZ2alwDpgWFLV8qBsD+4+w90r3b2yrCxlEhQRkf0Q5l1MBvwRWOruN7VRZ3BQDzObFMSzBXgdGGVmI8wsH7gAeDisWEVEZE9hXmI6EbgYWGRm84Oy7wEVAO5+OzANuNLMokAdcIEnppeNmtlXgJlALnBnMDYhIiJpYt1puu/KykrXGISISMeZ2Vx3r0y1TU9Si4hISkoQIiKSkhKEiEgWW7J+B88vrwrl2EoQIiJZ7C+z3+Wbf1sQyrGVIEREslgsHieSY6EcWwlCRCSLReNOrhKEiIi0Fou7ehAiIrIn9SBERCSlWMyJ5ITzVa4EISKSxdSDEBGRlDbXNhDWhElpWTBIRETCMX/N9tCOrR6EiEiWCnuyVSUIEZEsFYsrQYiISArRIEEcMrAolOMrQYiIZKl4cInpokkVoRxfCUJEJEs19yB0m6uIiOwmHiSIHFOCEBGRJM2D1JFcJQgREUkSy9YehJkNM7NnzWyJmS02s6+lqPM5M1toZovM7GUzG5+07Z2gfL6ZzQkrThGRbBULBqnDms01zCepo8BV7j7PzPoAc83sSXdfklRnNXCKu28zszOAGcBxSdtPdffNIcYoIpK1orGgB5FtCcLdNwAbgvc1ZrYUGAosSarzctIus4HysOIREelumm9zzc22S0zJzGw4MBF4tZ1qlwKPJX124Akzm2tm08OLTkQkO4U9SB36ZH1mVgw8AHzd3Xe0UedUEgnipKTik9x9nZkNAp40s7fcfVaKfacD0wEqKsJ5WEREpCvK2kFqADPLI5Ec/uruD7ZRZxzwB+Bsd9/SXO7u64L/bgIeAial2t/dZ7h7pbtXlpWVdXYTRES6rHXb64DwBqnDvIvJgD8CS939pjbqVAAPAhe7+/Kk8t7BwDZm1hs4HXgzrFhFRLLR5+96HSAr14M4EbgYWGRm84Oy7wEVAO5+O3AtMBC4LZFPiLp7JXAQ8FBQFgHucffHQ4xVRCRrVdc1hXLcMO9iehFot9/j7pcBl6UoXwWM33MPERFprb4pFspx9SS1iEiWa4rFQzmuEoSISJZrjCpBiIhIkiMG9wFgyuhBoRw/9OcgREQkHEcO6cvOxihjh5aEcnz1IEREslRTLE5eTnhf40oQIiJZKhrz0KbZACUIEZGsFY3HiagHISIirTXFnDz1IEREpLVoPE4kVz0IERFppSnmoU3UB0oQIiJZKxqLk6cehIiItNYYi1MQUYIQEZFWGpriFOQpQYiISCsN0TgFkdzQjq8EISKSpRqiMV1iEhGRPSV6EEoQIiLSSmIMQpeYREQkibvrEpOIiOwpGnfiTnYmCDMbZmbPmtkSM1tsZl9LUcfM7GYzW2FmC83s6KRtl5jZ28HrkrDiFBHJRg3BKnJh3sUU5oJBUeAqd59nZn2AuWb2pLsvSapzBjAqeB0H/A44zswGANcBlYAH+z7s7ttCjFdEJGs0NMUAsvM5CHff4O7zgvc1wFJgaKtqZwN/9oTZQD8zGwJ8HHjS3bcGSeFJYGpYsYqIZJvXVm8FYGN1fWjnSMsYhJkNByYCr7baNBRYk/R5bVDWVrmIiAD3z0l8Rc56uyq0c4SeIMysGHgA+Lq77wjh+NPNbI6ZzamqCu8HJSLSlRxV3g+AyaPKQjtHqAnCzPJIJIe/uvuDKaqsA4YlfS4Pytoq34O7z3D3SnevLCsL7wclItKVjBpUDMBnJoZ3cSXMu5gM+COw1N1vaqPaw8C/B3czHQ9Uu/sGYCZwupn1N7P+wOlBmYiIAHF3AHJDXA8izLuYTgQuBhaZ2fyg7HtABYC73w48CpwJrAB2AV8Itm01sxuA14P9rnf3rSHGKiKSVaKxLE4Q7v4i0G7k7u7Al9vYdidwZwihiYhkvVgaehB6klpEJAvF4koQIiKSQkuCMCUIERFJko5BaiUIEZEslI5BaiUIEZEs1NyDyFGCEBGRZNFgDCKiBCEiIsmaB6lzNEgtIiLJ4upBiIhIKlE9ByEiIqnE3TED0yUmERFJFot7qJeXQAlCRCQrxeIe6gA1KEGIiGSlWNxDHX8AJQgRkawUVYIQEZFU4q4EISIiKWiQWkREUtIgtYiIpKRBahERSUkJQkREUoqlYZA6EtaBzexO4BPAJncfm2L7t4DPJcVxJFDm7lvN7B2gBogBUXevDCtOEZFsFIt7qMuNQrg9iLuBqW1tdPdfuPsEd58AfBd43t23JlU5Ndiu5CAi0kpWX2Jy91nA1r1WTLgQuDesWEREupusThAdZWZFJHoaDyQVO/CEmc01s+mZiUxEpOtqjMWJ5GbpGMQ++CTwUqvLSye5+zozGwQ8aWZvBT2SPQQJZDpARUVF+NGKiHQB67bVMaK0d6jnyHgPAriAVpeX3H1d8N9NwEPApLZ2dvcZ7l7p7pVlZWWhBioi0lU0xeIU5uWGeo6MJggzKwFOAf4vqay3mfVpfg+cDryZmQhFRLqmppiTlxvuV3iYt7neC0wBSs1sLXAdkAfg7rcH1T4NPOHuO5N2PQh4KFglKQLc4+6PhxWniEg2aorFycvWMQh3v7ADde4mcTtsctkqYHw4UYmIdA91jbHQB6m7whiEiIjsgwVrtlPTEOXllVtCPY8ShIhIllmwdjsAq6p27qXmgVGCEBHJMmFP891ynrScRUREOk1JrzwAxg/rF+p5lCBERLJMaXEBAN/++OhQz6MEISKSZdwdQEuOiojI7uKJ/ECOEoSIiCSLBz2IsIeqlSBERLJM0IHAusKCQWb2NTPrawl/NLN5ZnZ6qJGJiEhKzT2IkK8wdbgH8UV330Fi4rz+wMXAT0OLSkRE2tQ8SN0lehB8cKnrTOAv7r6Y8C9/iYhICt48SN1FehBzzewJEgliZjAddzy8sEREpC0tdzGF3IPo6GyulwITgFXuvsvMBgBfCC8sERFpS8tdTF2kB3ECsMzdt5vZvwE/AKrDC0tERNrSMgYR8pX+jiaI3wG7zGw8cBWwEvhzaFGJiEibWsYgQn5QoaOHj3oiZZ0N/NbdbwX6hBeWiIi0pauNQdSY2XdJ3N462cxyCJYPFRGR9NrZEAW6zl1M5wMNJJ6H2AiUA78ILSoREWnTtx9YCEBTzPdS88B0KEEESeGvQImZfQKod/d2xyDM7E4z22Rmb7axfYqZVZvZ/OB1bdK2qWa2zMxWmNnV+9AeEZEeo6Y+GurxOzrVxnnAa8C5wHnAq2Y2bS+73Q1M3UudF9x9QvC6PjhXLnArcAYwBrjQzMZ0JE4RkZ7g4JJCAIaXFoV6no6OQXwfONbdNwGYWRnwFPD3tnZw91lmNnw/YpoErHD3VcG57iMxOL5kP44lItLtTB5VxnPLNzGoT2Go5+noGEROc3IIbNmHfdtzgpktMLPHzOxDQdlQYE1SnbVBmYiIAHVNMYryO/r3/f7r6BkeN7OZwL3B5/OBRw/w3POAQ9y91szOBP4BjNrXg5jZdGA6QEVFxQGGJCLStbk7Dy9Yn5ZzdXSQ+lvADGBc8Jrh7t85kBO7+w53rw3ePwrkmVkpsA4YllS1PChr6zgz3L3S3SvLysoOJCQRkS4v7DuXknW4j+LuDwAPdNaJzWww8L67u5lNIpGstgDbgVFmNoJEYrgAuKizzisiks1iwVNyA3vnh36udhOEmdXwweJFu20C3N37trPvvcAUoNTM1gLXETxc5+63A9OAK80sCtQBFwRPa0fN7CvATCAXuDOYXlxEpMeLxhMTaV9xyqGhn6vdBOHu+z2dhrtfuJftvwV+28a2RznwMQ4RkW6nuQeRG/Zj1GhNahGRrNKcICK5ShAiIpJEPQgREUkp2tyDUIIQEZFkC9cm1mpbVbUz9HMpQYiIZJFn30pMavHGe9tDP5cShIhIFjl2xAAArv1k+HOYKkGIiGSRv8x+F4DBJeFO1AdKECIiWSMWdxasSVxaKi4If7I+JQgRkSxx/5wPJrouiIT/9a0EISKSJWqTVpAz022uIiIS6B9M0Pd/Xz4xLedTghARyRJNscREfQf1DX+AGpQgRESyxvrtdQDkpWEeJlCCEBHJGrc8swKAvDQMUIMShIhI1ilOw3rUoAQhIpIVGqOJ8YePHjGInDRM1AdKECIiWeHGx5YC0BRP35rUShAiIlmg+c6laz9xZNrOqQQhIpIFeuXlAtC/KD9t51SCEBHJAs3PQERy0/e1HdqZzOxOM9tkZm+2sf1zZrbQzBaZ2ctmNj5p2ztB+XwzmxNWjCIi2aJ5Jbl0PQMB4fYg7gamtrN9NXCKux8F3ADMaLX9VHef4O6VIcUnIpI13lyXWEkukpO+HkRoN9O6+ywzG97O9peTPs4GysOKRUQk2/1z4Qag+/Qg9sWlwGNJnx14wszmmtn0DMUkItLlpGMW12bpeRyvHWZ2KokEcVJS8Unuvs7MBgFPmtlb7j6rjf2nA9MBKioqQo9XRCSdVmyq4cr/mQfA2RMOTuu5M9qDMLNxwB+As919S3O5u68L/rsJeAiY1NYx3H2Gu1e6e2VZWVnYIYuIpE00FudjN83i7U21ABwxuG9az5+xBGFmFcCDwMXuvjypvLeZ9Wl+D5wOpLwTSkSku9pUU8/xNz69W9nowcVpjSG0S0xmdi8wBSg1s7XAdUAegLvfDlwLDARuC66pRYM7lg4CHgrKIsA97v54WHGKiHQFtQ1Rxl43s906Rw5Jbw8izLuYLtzL9suAy1KUrwLG77mHiEj39c8F61OWTxldxqqqnby3dRdDSnqlNaaMD1KLiAi8s2VXyvKffXYckRxjQ3V9miNSghARyYitOxvpWxjBgVuefpvbn19Jn4IIvz5/AgV5OQwp6UWfwkjLJH0DiwvSHqMShIhImr3x3jY+fdvLe5TXNET52JiDMhBRal3lQTkRkR5jZdXOlOU/OCt9U3l3hBKEiMherN22iyXrdwCwZusu3lxXTXw/F+5pjMb55t8WAHDDOWNbylffeCaXTR554MF2Il1iEhFpJRZ3zr71Rc48agifHHcwU/97FjsbY7vVGV9ewq/Pn8CWnY3saoxxyZ2vAfC54yq49pNjKIjk7nHcp5e+z6V/+mCC6n87roKDSwpZVbUzrVNodJS5p2/5urBVVlb6nDmaHVxE2nf/nDXc99p7XDipgmnHlO/25XzTE8u4+ZkV7e4/tF8vquuaqG2Iptw+saIfl5wwnE+NP5jGWJx7X3uPv776HiuCJ6IBnr7qFA4tS++Db6mY2dy2Zs1WghCRbuXNddXE3Rk9uA9rttbxyVteJD+SQ3VdU8r6nxp/MDedN55bn13Jr59anrLOJSccwnfPPJJIjrUs2POX2e9yzT8+mOThqKEl3Df9eO56aTW/fCL1cQDOGjeEWy86+gBa2LmUIESky6uqaWDd9joOGVBEcWGEhWurWbstcb3/jhdWc80nxnB0RT+eXVbFzU+/3bJfjkHzcEDv/Nw9LgW1pU9BhJo2egAAS67/OEX5EdZvr2NISeEel4CaYnFGff8xTj68jD9/cffp4pZtrOGTt7xIY7AKXLO7vnAsp44e1KH40kUJQkS6nF2NUU7++bO4Q1mfAt7aWBPauX7+2XG8t3UX3zjtcAzY2RilT2Eei9dXc9bNL7bUu+XCicTizplHDSE/svd7eOJxJyen7bGD+qYY989Zw2ePLqd3Qdcc8lWC2Iu123Zx05PLWb+9jknDBzCgdz75kVxKi/MpLohQUpTH6s07qW9K/DXQ/L9DLO40xeMcVlbMwOIC3J3GWJxdjTHcoTAvh4JILkX5uexqjFHfFCOSaxREcnCHHfVN1DfFMQPDGFicjzts39XI9romauqjlBbnU5QfoSCSw6C+BeTn5lDSKy+t69KKHCh35+anV/DSis2YwdihJbz49maWvZ9ICqMP6sNR5SUUF0TYWF3Pyys3M3XsYCqHD+Dbf1+427GOrujHd6YewXEjB1JV00BpcT476qP0KYiQk2PU1DfRpzCv5bx7G/yNx51pt7/MqaMH8dWPjgrnB9CFKUG0o3pXEyf+7Jk2B5u6qsK8HAb1KaQgkkNDNE4kx6hrilHbEKUgkktuTqLb7e4t3W93JxZPXJsdPrA3ZlDSK4/CvFzqm2LUNsQoiOSwfVcjb22sYURpb8YM6UvFwCIG9y0kP5JDbUOUceX9yM/NoTAvh12NMV5bvZVo3BnarxdxdxauraZiQBFD+/diRGlv3J01W+vIi1ja55KRzFpVVctHfvX8bmXNfzA1m/uDj7X7lPDabbvoX5RPUX5ul7zTJ9u1lyC6Zp8njUqK8rjhnA9xTMUAqmrrOXJIX3Y2JP7ar65rYnNtA3WNMcr7F1HSK69lP8fJCf5nXbSummjcyTGI5BhF+Ykfa2M0Tn00xq7GGEX5uRRGconG4zTGEt/YfQsj9MrLJRZ8g2/d1QhAv1759CvKo3dBhC21DdQ3xalrilFV00BjNMb2uiZ2NcZYt62OWNwpzMshGncKIrn0KYxQ35ToweTkJFafav4nFY05ZvD2plqeX15FYyzOzoYosXgiieTn5hDJNeqbYgwf2JsFa7bz2Jsb2/zZlRbnU98Ubze5jiztzdrtdTRGE72vo4aWtPzMJo8qZeHaasYO7cvxIwayvrqe93fUc8wh/Zk6djCHlhUTizuPLtrAjvomTjvyIAYF0w5I1/bPhev5yj1v7FY2rryE3198DENKerGppp4LZszmjLGD9zqFRHn/ojBDlXb0+B6EtG9nQ5QN1XVsqK5nZ0OU3gUR5r27na07G9hRHyXHjJMPL2VA73yWrN9B/6J8jhjSh5r6KE8ueZ+7X36HykP6U1pcQHVdE6s372TjjsSkY30LI/TvnU9hJLflUsPeFERymDCsH/2K8jhpVBnTji6nV/6e95tL5tz67Ap+MXMZAN/42OEcOaQPp39ocIajkrboEpN0KfVNMfJzc3Yb3NtYXU/cnSElhazdVsff5qzhgXnrWLe9jgnD+nHiYQMpjOTyzpZdLHt/B+u317N1Z6LH9bWPjmLaMeUMG6C/NDPtp4+9xe3PrwTgtxdN5BPj0rtEpuw7JQjJWtFYPOWAfCzu/Oap5S0PNEVyjD99cRInHlaa7hCFxPjWTx5dyh0vrAZg3jWnMaB3foajko5oL0HoVhjp0tq6Wys3x/h/p4/mnZ+exT++fCIFkRy+fM88VlbVpqwv4YnG4i3JYWRpb56+6hQlh25CCUKy3oRh/XjsaycTizvn/352y6C/pMd/3vcGd7ywms9MHNplpo+QzqEEId1CxcAirjlrDJtrG7jt2fbn0ZHOs6mmnkcXbWRg73x+Pm2cbkPtZkJNEGZ2p5ltMrM329huZnazma0ws4VmdnTStkvM7O3gdUmYcUr3cG5lOWOH9uXW51ZQVdOQ6XC6vW07G/nkLYmnkH90zlg9vNkNhf0bvRuY2s72M4BRwWs68DsAMxsAXAccB0wCrjOz/qFGKlnPzLj5gonUN8W577X3Mh1Ot/fDRxazdWcjf/riJM44akimw5EQhJog3H0WsLWdKmcDf/aE2UA/MxsCfBx40t23uvs24EnaTzQiAIwsK+a4EQO457X3iLaaKE06zzNvvc//zV/P5ZNHcsrhZZkOR0KS6T7hUGBN0ue1QVlb5SJ7dV7lMDZU13PY9x9jY3V9psPplm5/fhWFeTl85SOHZToUCVGmE8QBM7PpZjbHzOZUVVVlOhzpAiaP+uBZiDtfWp3BSLqntzbu4LXVWznpsNKWaWWke8p0glgHDEv6XB6UtVW+B3ef4e6V7l5ZVqaursCgvoW8/eMzAJj37rYMR9P9vLxiCwCXntS11k+WzpfpBPEw8O/B3UzHA9XuvgGYCZxuZv2DwenTgzKRDsnLzeHyySOY8+42Xnx7c6bD6TZqG6L8+snlHDu8P8eNGJDpcCRkYd/mei/wCjDazNaa2aVmdoWZXRFUeRRYBawA7gC+BODuW4EbgNeD1/VBmUiHXTklcX38d8/ruYjO8tSS96lpiPLtqUe0u1COdA+hXkB09wv3st2BL7ex7U7gzjDikp5hQO98LjtpBH94cTWjf/AYv73oaE4bc1Cmw8pq/1y4nsF9CzmmQned9wSZvsQkEqrzjk0MZTVE43zvoUUZjia7xeLOrLcTK72p99AzKEFIt3b4QX249/LjAaiqaaCmvinDEWWv9cHCT0cM7pPpUCRNlCCk2zvh0IF8acqhADy3TLdC76/Vm3cCMLy0d4YjkXRRgpAe4RunHQ7AV+99g+60Bko6NSeIEUoQPYYShPQIebk5DOqTWPv4108uz3A02emlFZs5qG9By89Ruj8lCOkxnv/WqQC8tHJLhiPJPjX1TTy3vIozjxqiKb17ECUI6TF65edywsiBzH13G6+u2sK7W3ZmOqSs8drqrTRG45w+ZnCmQ5E0UoKQHuXyk0cAcP6M2Zzyi+eoa4xlOKLssGJTYinXMUP6ZjgSSSclCOlRPnLEQXxm4gcTA7+6WpebOmJV1U5KiwsoKcrLdCiSRkoQ0uP88tzx/PLc8eQYzFqueZo6YkVVLSPLdPdST6MEIT1OTo4x7ZhyTjm8jIcXrNdlpr1wd1ZsquWwQcWZDkXSTAlCeqxzK4exubaBI699nMff3JjpcLqsLTsbqa5r4tAyJYieRglCeqwPHzqw5f0V/zM3g5F0bSuDAWr1IHoeJQjpsfoV5fP41yfTfFv/V+99A4Dl79dQ+aMnWbN1FwD/9chifvPU25kKM+M27kgs2zq0X68MRyLppgQhPdoRg/vyxjWnAfDIgvUMv/pfXPanOWyubWTyz5/lsUUbuOuld/j1U8tZVVXbI6fpqKmPAtC3UMuL9jT6jUuP168on4e+9GE+fdvLALwX9BwArvzrvJb3H/nV8y3v83KNZTec0dL76M5PF9c2JBJEsRJEj6PfuAgwsaI/R1f0Y95721Nu//ChA3k5aYqOppgz8nuPtny+8TNHceGkitDjbObuzF+znZr6KKMH9+GgvoUt26KxOKOveZxY3Lng2GFMP3kkIw9ggLm2PkpujtErL7czQpcsogQhEnjwSycC8MOHFzOuvITjRg7kkjtf4+LjD+HCSRW8u2Unf3rlHf5n9nt77PvdBxdx/MiBaZnp9DO3vbRHIpv7g48RjTvz3t3G2KElxOKJS2H3vb6G+15fwzs/PWu/z1dT30Tv/Nxu3UuS1Kw7XVOtrKz0OXPmZDoM6QHmvLOVabe/wifGDWFl1U6Wbtix2/Zjh/fnb1d8uM396xpjOE5RfoRdjVHuefU9xhzcl0nDB3DLMyvYWF3Pz6aN22O/xxZt2O2yV3vGlZewcG01AK989yMMKdm/QebP3/UaG6vrefzrJ+/X/tK1mdlcd69MtU09CJH9UDl8wG5/lQ+/+l+7bX/9nW1843/nM3xgb+6fs4ZZ3z6V3GCZzqZYnPHXP0FjNM5904/nFzOXMffdbQB8/sPDufvldwA4t7KcCcP6Ecn94F6S5uQwbEAvPj2xnLOOGsLH/3tWyhh/c8FE1m7bxcV/fI1HF23k0pNG7FMb39m8k8/94VXWba/jvMryfdpXuodQ72Iys6lmtszMVpjZ1Sm2/9rM5gev5Wa2PWlbLGnbw2HGKXKg7v+PExjarxefTprn6aE31vHrp5azbnsdi9dXt5R/8pYXaYzGAbhgxuyW5ADw7LJNLe+n3f4KY384k1dXbSEai+92vmeumsL/O+1wRict/3l0RT+e/eYUAHIMhg8sYvKoMgb1KeB3z61k0dpq9mbd9jpeW70VgCm/fI512+sAOOOoIR39UUg3EtolJjPLBZYDpwFrgdeBC919SRv1vwpMdPcvBp9r3X2fRtZ0iUm6gnjcOfPmF3hrY81u5cePHMCS9TvYEdw2WpSfy65gmo+fTxvH2m113Px06uctxgzpy9+uOIEPXTeTb318NF8+9bCWbf/1yGLueukdnv/WFA4ZuOcYyL/94VVeXJGYc6q9sYimWJxR338MgDeuOY2JNzzZsm3J9R+nKF8XHLqj9i4xhdmDmASscPdV7t4I3Aec3U79C4F7Q4xHJC1ycozHv34yz31zCo985aSW8tmrtrYkhweuPIFfTBvfsu3Dhw7kxKQnu38+bRxnJf3VvmTDjpbpQI4c8kGvAeB7Zx7JI185KWVyAJg8qvSD4z7+VptxP5+0Xvcvn1gGwNVnHMFjX5us5NBDhZkghgJrkj6vDcr2YGaHACOAZ5KKC81sjpnNNrNzwgtTJBzDS3tzVHkJq35y5m7lpcUFHHPIAM4aN4SVPzmT+deeRnn/IiZW9G+pc86Eodz6uaNZ9qOplPdPDC5f9bcFAIwv77fb8fJycziqvKTNOC6bPJIfnHUkALc9t7LNh/1WVtW2vP/rq4k7tU46rJQjtQZEj9VV/iy4ANxH44UAAArNSURBVPi7uydPq3mIu68zs5HAM2a2yN1Xtt7RzKYD0wEqKtJ3H7pIR+XkGG/dMJWmWJw+hbuvp5CbY/QrygcgP5LDvZcfT3FBhPxI4m+3gkguT3zjZMZcO7Nln4HF+7YmdG6OcdnkkfzoX0sBWLV5524T723f1ciE6xOXkwb1KWBTTUPLNs2/1LOF2YNYBwxL+lwelKVyAa0uL7n7uuC/q4DngImpdnT3Ge5e6e6VZWVlBxqzSCgK83L3SA6pnHDowD16A8mXd8a101PYm9v/7RgAnn1r027ld7ywquV9/6J8/vnVkygtLuCuLxxLoR6O69HC7EG8DowysxEkEsMFwEWtK5nZEUB/4JWksv7ALndvMLNS4ETg5yHGKtKlvfidU5m9aiufGn/wfh9j6tjBHNS3gB/9aymbaxu5+owjALj12Q865sWFEcYOLWHODz52wDFL9gstQbh71My+AswEcoE73X2xmV0PzHH35ltXLwDu890vjB4J/N7M4iR6OT9t6+4nkZ6gvH8R044pOuDjHDmkL+/vqOL251eyeH31brflfuHE4Xz2aD3vIB/Qk9QiPUhVTQPH/vipPcqnHVPOL88dn2IP6e4ydZuriHQxZX0K+O1Few7nXXPWmAxEI12dEoRID/PRIw7iMxOH8sxVp3Dy4WXcc/lxlBTtfQBdeh5dYhIR6cF0iUlERPaZEoSIiKSkBCEiIikpQYiISEpKECIikpIShIiIpKQEISIiKSlBiIhISt3qQTkzqwK2A8mL75YkfU71vhTYfICnTj7u/tZra1vr8vY+Z1v7OlKm9mVv+zrS1u7YvuTybGjfIe6eeq0Ed+9WL2BGW59TvScxs2ynnnN/6rW1rb32ZHv7OlKm9mVv+zrS1u7YvlZ1sqp9rV/d8RLTI+18but9Z59zf+q1ta299rT+nG3t60iZ2nfgMtW+jrb1QHW19nVm2/bleJ3Rvt10q0tM+8PM5ngb85B0B2pfdlP7slu2t6879iD21YxMBxAytS+7qX3ZLavb1+N7ECIikpp6ECIikpIShIiIpKQEISIiKSlBtMPMJpvZ7Wb2BzN7OdPxdDYzyzGzH5vZLWZ2Sabj6WxmNsXMXgh+h1MyHU8YzKy3mc0xs09kOpbOZmZHBr+7v5vZlZmOp7OZ2TlmdoeZ/a+ZnZ7peFLptgnCzO40s01m9mar8qlmtszMVpjZ1e0dw91fcPcrgH8Cfwoz3n3VGe0DzgbKgSZgbVix7o9Oap8DtUAh3bN9AN8B7g8nyv3XSf/+lgb//s4DTgwz3n3VSe37h7tfDlwBnB9mvPur297FZGYnk/hy+LO7jw3KcoHlwGkkvjBeBy4EcoEbWx3ii+6+KdjvfuBSd69JU/h71RntC17b3P33ZvZ3d5+Wrvj3ppPat9nd42Z2EHCTu38uXfHvTSe1bzwwkEQC3Ozu/0xP9HvXWf/+zOxTwJXAX9z9nnTFvzed/P3yK+Cv7j4vTeF3WCTTAYTF3WeZ2fBWxZOAFe6+CsDM7gPOdvcbgZRddDOrAKq7UnKAzmmfma0FGoOPsfCi3Xed9fsLbAMKwohzf3XS728K0BsYA9SZ2aPuHg8z7o7qrN+fuz8MPGxm/wK6TILopN+fAT8FHuuKyQG6cYJow1BgTdLntcBxe9nnUuCu0CLqXPvavgeBW8xsMjArzMA6yT61z8w+A3wc6Af8NtzQOsU+tc/dvw9gZp8n6C2FGt2B29ff3xTgMySS+6OhRtY59vXf31eBjwElZnaYu98eZnD7o6cliH3m7tdlOoawuPsuEgmwW3L3B0kkwW7N3e/OdAxhcPfngOcyHEZo3P1m4OZMx9GebjtI3YZ1wLCkz+VBWXeh9mU3tS+7dbv29bQE8TowysxGmFk+cAHwcIZj6kxqX3ZT+7Jbt2tft00QZnYv8Aow2szWmtml7h4FvgLMBJYC97v74kzGub/UPrWvK1P7srt9zbrtba4iInJgum0PQkREDowShIiIpKQEISIiKSlBiIhISkoQIiKSkhKEiIikpAQhGWNmtWk4xxVm9u9hn6fVOc8xszH7ud+1wfsfmtk3Oz+6fWeJdTXanSnWzI4ys7vTFJKkieZikqxnZrnunnI22rAmQGvvnMA5JNYQWbKPh/028KkDCixD3H2RmZWbWYW7v5fpeKRzqAchXYKZfcvMXjezhWb2X0nl/zCzuWa22MymJ5XXmtmvzGwBcELw+cdmtsDMZgdrQOz2l7iZPWdmPzOz18xseTCLLWZWZGb3m9kSM3vIzF41s8oUMb4T7D8PONfMLg9iXmBmDwTH+TCJL/lfmNl8Mzs0eD0etOMFMzsixbEPBxrcfXOKbROCNi0M4usflB8blM03s19Yq8VrgjpDzGxWUOfNpDZPNbN5QexPB2WTzOwVM3vDzF42s9EpjtfbEovlvBbUOztp8yMkppeQbkIJQjLOEsstjiIxn/4E4BhLLMgCiYVVjgEqgf80s4FBeW/gVXcf7+4vBp9nu/t4ElOXX97G6SLuPgn4OtA8U++XSCycNAa4BjimnXC3uPvR7n4f8KC7HxuccymJRaVeJjH/zrfcfYK7rwRmAF8N2vFN4LYUxz0RaGtNgD8D33H3ccCipLjvAv7D3SfQ9noeFwEzgzrjgflmVgbcAXw2iP3coO5bwGR3nwhcC/wkxfG+DzwT/AxPJZEIewfb5gCT24hDspAuMUlXcHrweiP4XEwiYcwikRQ+HZQPC8q3kPhCfCDpGI0kLusAzCWxqlcqDybVGR68Pwn4DYC7v2lmC9uJ9X+T3o81sx+RWG+imMQcPLsxs2Lgw8DfzKy5ONXiRUOAqhT7lwD93P35oOhPwbH6AX3c/ZWg/B5SL7rzOnCnmeUB/3D3+ZZYZ2GWu68O2rw1qFsC/MnMRpFYrjUvxfFOBz6VND5SCFSQSJCbgINT7CNZSglCugIDbnT33+9WmPgi+xhwgrvvMrPnSHwhAdS3GgNo8g8mFovR9v/bDR2o056dSe/vBs5x9wWWWLRnSor6OcD24C/49tSR+ILuVMHKZycDZwF3m9lNJFbYS+UG4Fl3/7QlVkt7LkUdI9HzWJZiWyGJdkg3oUtM0hXMBL4Y/LWNmQ01s0EkvjC3BcnhCOD4kM7/EnBecO4xwFEd3K8PsCH46zx5veuaYBvuvgNYbWbnBsc3Mxuf4lhLgcNaF7p7NbCteewAuBh43t23AzVm1rxiWcpr/2Z2CPC+u98B/AE4GpgNnGxmI4I6A4LqJXywfsHn22jzTOCrFnSHzGxi0rbDgT3GQSR7KUFIxrn7EyQukbxiZouAv5P4gn0ciJjZUhJr984OKYTbgDIzWwL8CFgMVHdgv2uAV0kkmLeSyu8DvhUM4h5KInlcGgyoLwbO3uNIictpE5u/eFu5hMS1/oUkxmiuD8ovBe4ws/kkxmBSxTwFWGBmbwDnA79x9ypgOvBgEFPzZbOfAzcGddvqXd1A4tLTQjNbHHxudirwrzb2kyyk6b6lxzOzXCDP3euDL/SngNHu3pjmOH4DPOLuT3WwfrG71wbvrwaGuPvXwoyxnVgKgOeBk4J1EaQb0BiECBQBzwaXigz4UrqTQ+AntL/IfWtnmdl3Sfw7fpe2LwulQwVwtZJD96IehIiIpKQxCBERSUkJQkREUlKCEBGRlJQgREQkJSUIERFJSQlCRERS+v+9E8sVevbRsgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ty1Qe9VmNQfM",
        "colab_type": "text"
      },
      "source": [
        "If you pick a bad learning rate below, learner will remember the bad training. I've purposely included the model = and learner = cells to reset you"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "44SKAYOeCueD",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 196
        },
        "outputId": "fe3bdd6e-c78d-4972-bc86-731a5ff736ee"
      },
      "source": [
        "model = t.get_classifier()\n",
        "learner = ktrain.get_learner(model, train_data=trn, val_data=evalr, batch_size=6)\n",
        "learner.fit_onecycle(1e-4, 3)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "begin training using onecycle policy with max lr of 0.0001...\n",
            "Train for 1135 steps, validate for 71 steps\n",
            "Epoch 1/3\n",
            "1135/1135 [==============================] - 181s 159ms/step - loss: 0.8092 - accuracy: 0.5513 - val_loss: 0.7062 - val_accuracy: 0.6210\n",
            "Epoch 2/3\n",
            "1135/1135 [==============================] - 159s 140ms/step - loss: 0.5958 - accuracy: 0.7169 - val_loss: 0.5638 - val_accuracy: 0.7532\n",
            "Epoch 3/3\n",
            "1135/1135 [==============================] - 158s 139ms/step - loss: 0.3062 - accuracy: 0.8894 - val_loss: 0.4664 - val_accuracy: 0.8184\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f6832043240>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7hiutrV2PXIo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 261
        },
        "outputId": "a1e208c2-921f-4e2a-a83f-a779b900d606"
      },
      "source": [
        "model = t.get_classifier()\n",
        "learner = ktrain.get_learner(model, train_data=trn, val_data=evalr, batch_size=6)\n",
        "learner.fit_onecycle(1.2e-4, 5)"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "begin training using onecycle policy with max lr of 0.00012...\n",
            "Train for 1135 steps, validate for 71 steps\n",
            "Epoch 1/5\n",
            "1135/1135 [==============================] - 183s 161ms/step - loss: 0.8110 - accuracy: 0.5574 - val_loss: 0.6995 - val_accuracy: 0.6232\n",
            "Epoch 2/5\n",
            "1135/1135 [==============================] - 158s 140ms/step - loss: 0.6197 - accuracy: 0.6855 - val_loss: 0.5355 - val_accuracy: 0.7889\n",
            "Epoch 3/5\n",
            "1135/1135 [==============================] - 158s 140ms/step - loss: 0.3896 - accuracy: 0.8490 - val_loss: 0.5486 - val_accuracy: 0.8158\n",
            "Epoch 4/5\n",
            "1135/1135 [==============================] - 159s 140ms/step - loss: 0.1814 - accuracy: 0.9423 - val_loss: 0.4791 - val_accuracy: 0.8321\n",
            "Epoch 5/5\n",
            "1135/1135 [==============================] - 159s 140ms/step - loss: 0.0817 - accuracy: 0.9766 - val_loss: 0.5729 - val_accuracy: 0.8369\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f682acde240>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xOzrw8JwC4LJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66
        },
        "outputId": "c435dcd3-ba7c-41a4-f530-fcb13b17b279"
      },
      "source": [
        "learner.view_top_losses(n=1, preproc=t)"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----------\n",
            "id:1451 | loss:5.82 | true:neutral | pred:negative)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gqsiXkLFC9hR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "predictor = ktrain.get_predictor(learner.model, preproc=t)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}