Spam_Classification_BertTokenizer_Model.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Spam Classification BertTokenizer Model.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "57f81eff8bf0405e85dc849c1350a3c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_cc9ae7b4fe2f4a94943e99c4df402cf7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_1cf7011527b84e3ebe6ae8f997a6f72e",
              "IPY_MODEL_685a3c87ad0e4754801974e80798ea81"
            ]
          }
        },
        "cc9ae7b4fe2f4a94943e99c4df402cf7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1cf7011527b84e3ebe6ae8f997a6f72e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_bdd35a013d3d4f7e972139dc73810cdb",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 433,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 433,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_093acd3080574af3b7e1975d83412e9f"
          }
        },
        "685a3c87ad0e4754801974e80798ea81": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d522bf59bfb34761a1e96ccc73b43eba",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 433/433 [00:00&lt;00:00, 3.31kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c685cd4b922a4635a063dbe738333824"
          }
        },
        "bdd35a013d3d4f7e972139dc73810cdb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "093acd3080574af3b7e1975d83412e9f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d522bf59bfb34761a1e96ccc73b43eba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c685cd4b922a4635a063dbe738333824": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "800078c78d5c4d45a55bd6f1494c0f1a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8467c502206e42178a8cd5e40c25bfda",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4c2b76b9a25b4ba4a91ce5042aaa3a30",
              "IPY_MODEL_b8ba8c2d7f0f420d8ba8bfb425ebda7f"
            ]
          }
        },
        "8467c502206e42178a8cd5e40c25bfda": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4c2b76b9a25b4ba4a91ce5042aaa3a30": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_58365349615940018640198c6849ade4",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_560b67f7e6534851a03064dfae396299"
          }
        },
        "b8ba8c2d7f0f420d8ba8bfb425ebda7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8d6247528d964c92bdd4e69c924bdf7b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 440M/440M [00:34&lt;00:00, 12.9MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_53327844a1eb4316978a095ba226e1e5"
          }
        },
        "58365349615940018640198c6849ade4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "560b67f7e6534851a03064dfae396299": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8d6247528d964c92bdd4e69c924bdf7b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "53327844a1eb4316978a095ba226e1e5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e09cf65579af46bfac60c756bc611255": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_f8f67296205c404bb72695b910462a49",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e72feddbfe21411cb57a58044fcf9013",
              "IPY_MODEL_deb4b1348bb446d28d90f156aa3edd5a"
            ]
          }
        },
        "f8f67296205c404bb72695b910462a49": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e72feddbfe21411cb57a58044fcf9013": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_402bf67f3d1b4aa5bb8dcf558507ad7e",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_fee4cc110ca947d798d417612ae12fec"
          }
        },
        "deb4b1348bb446d28d90f156aa3edd5a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_57ed2206a65a49c989d4894b201a182a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 453kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c47394edf17941d697b968542b4323ea"
          }
        },
        "402bf67f3d1b4aa5bb8dcf558507ad7e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "fee4cc110ca947d798d417612ae12fec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "57ed2206a65a49c989d4894b201a182a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c47394edf17941d697b968542b4323ea": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/pymacbit/Spam-Classification-BertTokenizer-Model/blob/master/Spam_Classification_BertTokenizer_Model.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OFOTiqrtNvyy",
        "colab_type": "text"
      },
      "source": [
        "# Spam Classification Bert Model \n",
        "\n",
        "Spam classifier BERT Model to classify and Predict Text as spam or not spam.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1hkhc10wNrGt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 615
        },
        "outputId": "a956a43a-92fa-47eb-c03f-56bfeb4b8271"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/27/3c/91ed8f5c4e7ef3227b4119200fc0ed4b4fd965b1f0172021c25701087825/transformers-3.0.2-py3-none-any.whl (769kB)\n",
            "\r\u001b[K     |▍                               | 10kB 23.8MB/s eta 0:00:01\r\u001b[K     |▉                               | 20kB 6.3MB/s eta 0:00:01\r\u001b[K     |█▎                              | 30kB 7.4MB/s eta 0:00:01\r\u001b[K     |█▊                              | 40kB 8.3MB/s eta 0:00:01\r\u001b[K     |██▏                             | 51kB 5.7MB/s eta 0:00:01\r\u001b[K     |██▋                             | 61kB 5.9MB/s eta 0:00:01\r\u001b[K     |███                             | 71kB 5.7MB/s eta 0:00:01\r\u001b[K     |███▍                            | 81kB 5.7MB/s eta 0:00:01\r\u001b[K     |███▉                            | 92kB 5.8MB/s eta 0:00:01\r\u001b[K     |████▎                           | 102kB 5.8MB/s eta 0:00:01\r\u001b[K     |████▊                           | 112kB 5.8MB/s eta 0:00:01\r\u001b[K     |█████▏                          | 122kB 5.8MB/s eta 0:00:01\r\u001b[K     |█████▌                          | 133kB 5.8MB/s eta 0:00:01\r\u001b[K     |██████                          | 143kB 5.8MB/s eta 0:00:01\r\u001b[K     |██████▍                         | 153kB 5.8MB/s eta 0:00:01\r\u001b[K     |██████▉                         | 163kB 5.8MB/s eta 0:00:01\r\u001b[K     |███████▎                        | 174kB 5.8MB/s eta 0:00:01\r\u001b[K     |███████▊                        | 184kB 5.8MB/s eta 0:00:01\r\u001b[K     |████████                        | 194kB 5.8MB/s eta 0:00:01\r\u001b[K     |████████▌                       | 204kB 5.8MB/s eta 0:00:01\r\u001b[K     |█████████                       | 215kB 5.8MB/s eta 0:00:01\r\u001b[K     |█████████▍                      | 225kB 5.8MB/s eta 0:00:01\r\u001b[K     |█████████▉                      | 235kB 5.8MB/s eta 0:00:01\r\u001b[K     |██████████▎                     | 245kB 5.8MB/s eta 0:00:01\r\u001b[K     |██████████▋                     | 256kB 5.8MB/s eta 0:00:01\r\u001b[K     |███████████                     | 266kB 5.8MB/s eta 0:00:01\r\u001b[K     |███████████▌                    | 276kB 5.8MB/s eta 0:00:01\r\u001b[K     |████████████                    | 286kB 5.8MB/s eta 0:00:01\r\u001b[K     |████████████▍                   | 296kB 5.8MB/s eta 0:00:01\r\u001b[K     |████████████▉                   | 307kB 5.8MB/s eta 0:00:01\r\u001b[K     |█████████████▏                  | 317kB 5.8MB/s eta 0:00:01\r\u001b[K     |█████████████▋                  | 327kB 5.8MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 337kB 5.8MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 348kB 5.8MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 358kB 5.8MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 368kB 5.8MB/s eta 0:00:01\r\u001b[K     |███████████████▊                | 378kB 5.8MB/s eta 0:00:01\r\u001b[K     |████████████████▏               | 389kB 5.8MB/s eta 0:00:01\r\u001b[K     |████████████████▋               | 399kB 5.8MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 409kB 5.8MB/s eta 0:00:01\r\u001b[K     |█████████████████▌              | 419kB 5.8MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 430kB 5.8MB/s eta 0:00:01\r\u001b[K     |██████████████████▎             | 440kB 5.8MB/s eta 0:00:01\r\u001b[K     |██████████████████▊             | 450kB 5.8MB/s eta 0:00:01\r\u001b[K     |███████████████████▏            | 460kB 5.8MB/s eta 0:00:01\r\u001b[K     |███████████████████▋            | 471kB 5.8MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 481kB 5.8MB/s eta 0:00:01\r\u001b[K     |████████████████████▌           | 491kB 5.8MB/s eta 0:00:01\r\u001b[K     |████████████████████▉           | 501kB 5.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████▎          | 512kB 5.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████▊          | 522kB 5.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████▏         | 532kB 5.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████▋         | 542kB 5.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 552kB 5.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████▍        | 563kB 5.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 573kB 5.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 583kB 5.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████▊       | 593kB 5.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▏      | 604kB 5.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▋      | 614kB 5.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 624kB 5.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▍     | 634kB 5.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▉     | 645kB 5.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▎    | 655kB 5.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▊    | 665kB 5.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▏   | 675kB 5.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▌   | 686kB 5.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 696kB 5.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▍  | 706kB 5.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▉  | 716kB 5.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 727kB 5.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▊ | 737kB 5.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 747kB 5.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▌| 757kB 5.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 768kB 5.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 778kB 5.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Collecting tokenizers==0.8.1.rc1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/40/d0/30d5f8d221a0ed981a186c8eb986ce1c94e3a6e87f994eae9f4aa5250217/tokenizers-0.8.1rc1-cp36-cp36m-manylinux1_x86_64.whl (3.0MB)\n",
            "\u001b[K     |████████████████████████████████| 3.0MB 19.7MB/s \n",
            "\u001b[?25hCollecting sentencepiece!=0.1.92\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 56.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 43.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.16.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893260 sha256=975a8997d57d02d2bf00e0463260af274e8174a2f0d383321492a792cadd2409\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sentencepiece, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.91 tokenizers-0.8.1rc1 transformers-3.0.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x4giRzM7NtHJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import classification_report\n",
        "import transformers\n",
        "from transformers import AutoModel, BertTokenizerFast\n",
        "\n",
        "# specify GPU\n",
        "device = torch.device(\"cuda\")"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kKd-Tj3hOMsZ",
        "colab_type": "text"
      },
      "source": [
        "## Load Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QQ30LbGh3c00",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "e500d0f7-32e2-4103-8b0d-92d589ccf771"
      },
      "source": [
        "!gdown --id 1NQQrRu8HrpIcYC2NKhgkStsTC4r2B-nK --output spamdata.csv"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1NQQrRu8HrpIcYC2NKhgkStsTC4r2B-nK\n",
            "To: /content/spamdata.csv\n",
            "\r  0% 0.00/467k [00:00<?, ?B/s]\r100% 467k/467k [00:00<00:00, 68.4MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cwJrQFQgN_BE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "e27156c6-d9f7-47b2-b3b1-6bb66f9a2eeb"
      },
      "source": [
        "df = pd.read_csv(\"spamdata.csv\")\n",
        "df.head()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>label</th>\n",
              "      <th>text</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>Ok lar... Joking wif u oni...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>U dun say so early hor... U c already then say...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   label                                               text\n",
              "0      0  Go until jurong point, crazy.. Available only ...\n",
              "1      0                      Ok lar... Joking wif u oni...\n",
              "2      1  Free entry in 2 a wkly comp to win FA Cup fina...\n",
              "3      0  U dun say so early hor... U c already then say...\n",
              "4      0  Nah I don't think he goes to usf, he lives aro..."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fzPPOrVQWiW5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "6a7b69c7-3c45-4c8f-eb6f-199d10162eee"
      },
      "source": [
        "df.shape"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(5572, 2)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "676DPU1BOPdp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "dbf22436-caab-48e7-db76-f5ccbaaad582"
      },
      "source": [
        "# check class distribution\n",
        "df['label'].value_counts(normalize = True)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    0.865937\n",
              "1    0.134063\n",
              "Name: label, dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MKfWnApvOoE7",
        "colab_type": "text"
      },
      "source": [
        "## Split train dataset into train, validation and test sets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mfhSPF5jOWb7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_text, temp_text, train_labels, temp_labels = train_test_split(df['text'], df['label'], \n",
        "                                                                    random_state=2018, \n",
        "                                                                    test_size=0.3, \n",
        "                                                                    stratify=df['label'])\n",
        "\n",
        "# we will use temp_text and temp_labels to create validation and test set\n",
        "val_text, test_text, val_labels, test_labels = train_test_split(temp_text, temp_labels, \n",
        "                                                                random_state=2018, \n",
        "                                                                test_size=0.5, \n",
        "                                                                stratify=temp_labels)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n7hsdLoCO7uB",
        "colab_type": "text"
      },
      "source": [
        "## Import BERT Model and BERT Tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S1kY3gZjO2RE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 164,
          "referenced_widgets": [
            "57f81eff8bf0405e85dc849c1350a3c2",
            "cc9ae7b4fe2f4a94943e99c4df402cf7",
            "1cf7011527b84e3ebe6ae8f997a6f72e",
            "685a3c87ad0e4754801974e80798ea81",
            "bdd35a013d3d4f7e972139dc73810cdb",
            "093acd3080574af3b7e1975d83412e9f",
            "d522bf59bfb34761a1e96ccc73b43eba",
            "c685cd4b922a4635a063dbe738333824",
            "800078c78d5c4d45a55bd6f1494c0f1a",
            "8467c502206e42178a8cd5e40c25bfda",
            "4c2b76b9a25b4ba4a91ce5042aaa3a30",
            "b8ba8c2d7f0f420d8ba8bfb425ebda7f",
            "58365349615940018640198c6849ade4",
            "560b67f7e6534851a03064dfae396299",
            "8d6247528d964c92bdd4e69c924bdf7b",
            "53327844a1eb4316978a095ba226e1e5",
            "e09cf65579af46bfac60c756bc611255",
            "f8f67296205c404bb72695b910462a49",
            "e72feddbfe21411cb57a58044fcf9013",
            "deb4b1348bb446d28d90f156aa3edd5a",
            "402bf67f3d1b4aa5bb8dcf558507ad7e",
            "fee4cc110ca947d798d417612ae12fec",
            "57ed2206a65a49c989d4894b201a182a",
            "c47394edf17941d697b968542b4323ea"
          ]
        },
        "outputId": "d2eff5af-1788-40b0-b7f7-984aa4db84e8"
      },
      "source": [
        "# import BERT-base pretrained model\n",
        "bert = AutoModel.from_pretrained('bert-base-uncased')\n",
        "\n",
        "# Load the BERT tokenizer\n",
        "tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "57f81eff8bf0405e85dc849c1350a3c2",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "800078c78d5c4d45a55bd6f1494c0f1a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=440473133.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e09cf65579af46bfac60c756bc611255",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_zOKeOMeO-DT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# sample data\n",
        "text = [\"this is a bert model tutorial\", \"we will fine-tune a bert model\"]\n",
        "\n",
        "# encode text\n",
        "sent_id = tokenizer.batch_encode_plus(text, padding=True, return_token_type_ids=False)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oAH73n39PHLw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "2f2cca48-83a8-46ee-ad3d-e28b1ade580b"
      },
      "source": [
        "# output\n",
        "print(sent_id)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'input_ids': [[101, 2023, 2003, 1037, 14324, 2944, 14924, 4818, 102, 0], [101, 2057, 2097, 2986, 1011, 8694, 1037, 14324, 2944, 102]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8wIYaWI_Prg8",
        "colab_type": "text"
      },
      "source": [
        "## Tokenization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yKwbpeN_PMiu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        },
        "outputId": "64fb670b-24be-4a30-c01d-47e027648ddc"
      },
      "source": [
        "# get length of all the messages in the train set\n",
        "seq_len = [len(i.split()) for i in train_text]\n",
        "\n",
        "pd.Series(seq_len).hist(bins = 30)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7fecfc889198>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD4CAYAAADrRI2NAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAUhklEQVR4nO3df5Dcd13H8efbxhboYdIfzE0niV7QiFMbleamrYMyd8aBNEVSFZl2OpBgnYxji8XWoUFG66jMBBURRsSJpkPQyhURprEtQgw9Gf5IpamlSVtKryVIbkIqtASPVjH69o/9nG7Pu9z+SHZv83k+Zm7uu5/vZ3df++32td/97vc2kZlIkurxXf0OIEnqLYtfkipj8UtSZSx+SaqMxS9JlVnW7wAnc+GFF+bIyEjb1/v2t7/Nueeee+oDnUaDlnnQ8oKZe2XQMg9aXlg884EDB76emS9bcEJmLtmf9evXZyfuu+++jq7XT4OWedDyZpq5VwYt86DlzVw8M/BAnqRbPdQjSZWx+CWpMha/JFXG4pekylj8klQZi1+SKmPxS1JlLH5JqozFL0mVWdJf2dArI9vvaWne4R1XneYkknT6uccvSZVZtPgj4vaIeDoiDjWN/UFEfDEiHo6IT0TEiqZ174iIqYh4PCJe2zS+sYxNRcT2U/9QJEmtaGWP/0PAxjlje4FLMvNHgC8B7wCIiIuBa4AfLtf504g4KyLOAj4AXAlcDFxb5kqSemzR4s/MzwLPzBn7dGaeKBf3A6vK8mZgIjP/IzO/DEwBl5Wfqcx8KjO/A0yUuZKkHovGN3guMiliBLg7My+ZZ93fAXdm5l9FxJ8A+zPzr8q6XcAny9SNmflLZfxNwOWZeeM8t7cN2AYwPDy8fmJiou0HNTMzw9DQUMvzD04fb2neupXL287SqnYz99ug5QUz98qgZR60vLB45vHx8QOZObrQ+q7O6omIdwIngDu6uZ1mmbkT2AkwOjqaY2Njbd/G5OQk7Vxva6tn9VzXfpZWtZu53wYtL5i5VwYt86Dlhe4zd1z8EbEVeB2wIf/vbcM0sLpp2qoyxknGJUk91NHpnBGxEXg78PrMfK5p1R7gmog4JyLWAGuBfwI+D6yNiDURcTaND4D3dBddktSJRff4I+IjwBhwYUQcAW6jcRbPOcDeiIDGcf1fzsxHIuKjwKM0DgHdkJn/VW7nRuBTwFnA7Zn5yGl4PJKkRSxa/Jl57TzDu04y/13Au+YZvxe4t610kqRTzr/claTKWPySVBmLX5IqY/FLUmUsfkmqjMUvSZWx+CWpMha/JFXG4pekylj8klQZi1+SKmPxS1JlLH5JqozFL0mVsfglqTIWvyRVxuKXpMpY/JJUGYtfkipj8UtSZSx+SaqMxS9JlbH4JakyFr8kVcbil6TKLFr8EXF7RDwdEYeaxs6PiL0R8UT5fV4Zj4h4f0RMRcTDEXFp03W2lPlPRMSW0/NwJEmLaWWP/0PAxjlj24F9mbkW2FcuA1wJrC0/24APQuOFArgNuBy4DLht9sVCktRbixZ/Zn4WeGbO8GZgd1neDVzdNP7hbNgPrIiIi4DXAnsz85nMfBbYy/9/MZEk9UBk5uKTIkaAuzPzknL5m5m5oiwH8GxmroiIu4Edmfm5sm4fcCswBrwoM3+vjP8m8Hxm/uE897WNxrsFhoeH109MTLT9oGZmZhgaGmp5/sHp4y3NW7dyedtZWtVu5n4btLxg5l4ZtMyDlhcWzzw+Pn4gM0cXWr+s2wCZmRGx+KtH67e3E9gJMDo6mmNjY23fxuTkJO1cb+v2e1qad/i69rO0qt3M/TZoecHMvTJomQctL3SfudOzeo6VQziU30+X8WlgddO8VWVsoXFJUo91Wvx7gNkzc7YAdzWNv7mc3XMFcDwzjwKfAl4TEeeVD3VfU8YkST226KGeiPgIjWP0F0bEERpn5+wAPhoR1wNfAd5Ypt8LbAKmgOeAtwBk5jMR8bvA58u838nMuR8YS5J6YNHiz8xrF1i1YZ65CdywwO3cDtzeVjpJ0innX+5KUmUsfkmqjMUvSZWx+CWpMha/JFXG4pekylj8klQZi1+SKmPxS1JlLH5JqozFL0mVsfglqTIWvyRVxuKXpMpY/JJUGYtfkipj8UtSZSx+SaqMxS9JlbH4JakyFr8kVcbil6TKWPySVBmLX5Iq01XxR8SvRcQjEXEoIj4SES+KiDURcX9ETEXEnRFxdpl7Trk8VdaPnIoHIElqT8fFHxErgV8FRjPzEuAs4Brg3cB7M/MHgGeB68tVrgeeLePvLfMkST3W7aGeZcCLI2IZ8BLgKPBTwMfK+t3A1WV5c7lMWb8hIqLL+5cktSkys/MrR9wEvAt4Hvg0cBOwv+zVExGrgU9m5iURcQjYmJlHyrongcsz8+tzbnMbsA1geHh4/cTERNu5ZmZmGBoaann+wenjLc1bt3J521la1W7mfhu0vGDmXhm0zIOWFxbPPD4+fiAzRxdav6zTO46I82jsxa8Bvgn8DbCx09ublZk7gZ0Ao6OjOTY21vZtTE5O0s71tm6/p6V5h69rP0ur2s3cb4OWF8zcK4OWedDyQveZuznU89PAlzPzXzPzP4GPA68CVpRDPwCrgOmyPA2sBijrlwPf6OL+JUkd6Kb4/wW4IiJeUo7VbwAeBe4D3lDmbAHuKst7ymXK+s9kN8eZJEkd6bj4M/N+Gh/SPggcLLe1E7gVuDkipoALgF3lKruAC8r4zcD2LnJLkjrU8TF+gMy8DbhtzvBTwGXzzP134Be6ub92jbR47F6SauJf7kpSZSx+SaqMxS9JlbH4JakyFr8kVcbil6TKWPySVBmLX5IqY/FLUmUsfkmqjMUvSZWx+CWpMha/JFXG4pekylj8klQZi1+SKmPxS1JlLH5JqozFL0mVsfglqTIWvyRVxuKXpMpY/JJUGYtfkipj8UtSZboq/ohYEREfi4gvRsRjEfHjEXF+ROyNiCfK7/PK3IiI90fEVEQ8HBGXnpqHIElqR7d7/O8D/j4zfwj4UeAxYDuwLzPXAvvKZYArgbXlZxvwwS7vW5LUgY6LPyKWA68GdgFk5ncy85vAZmB3mbYbuLosbwY+nA37gRURcVHHySVJHYnM7OyKET8G7AQepbG3fwC4CZjOzBVlTgDPZuaKiLgb2JGZnyvr9gG3ZuYDc253G413BAwPD6+fmJhoO9vMzAxDQ0McnD7e0WNbyLqVy0/p7TWbzTwoBi0vmLlXBi3zoOWFxTOPj48fyMzRhdYv6+K+lwGXAm/NzPsj4n3832EdADIzI6KtV5bM3EnjBYXR0dEcGxtrO9jk5CRjY2Ns3X5P29c9mcPXtZ+lVbOZB8Wg5QUz98qgZR60vNB95m6O8R8BjmTm/eXyx2i8EBybPYRTfj9d1k8Dq5uuv6qMSZJ6qOPiz8yvAV+NiFeUoQ00DvvsAbaUsS3AXWV5D/DmcnbPFcDxzDza6f1LkjrTzaEegLcCd0TE2cBTwFtovJh8NCKuB74CvLHMvRfYBEwBz5W5kqQe66r4M/MhYL4PEDbMMzeBG7q5P0lS9/zLXUmqjMUvSZWx+CWpMt1+uKsujDT9ncEt604s+HcHh3dc1atIkirgHr8kVcbil6TKWPySVBmLX5Iq44e7bRhp8Uvf/DBW0lLmHr8kVcbil6TKWPySVBmLX5IqY/FLUmU8q+c0aPXsH0nqB/f4JakyFr8kVcbil6TKWPySVBmLX5IqY/FLUmUsfkmqjMUvSZWx+CWpMl0Xf0ScFRH/HBF3l8trIuL+iJiKiDsj4uwyfk65PFXWj3R735Kk9p2KPf6bgMeaLr8beG9m/gDwLHB9Gb8eeLaMv7fMkyT1WFfFHxGrgKuAvyiXA/gp4GNlym7g6rK8uVymrN9Q5kuSeqjbPf4/Bt4O/He5fAHwzcw8US4fAVaW5ZXAVwHK+uNlviSphyIzO7tixOuATZn5KxExBvw6sBXYXw7nEBGrgU9m5iURcQjYmJlHyrongcsz8+tzbncbsA1geHh4/cTERNvZZmZmGBoa4uD08Y4eWz8MvxiOPT//unUrl/c2TAtmt/EgMXNvDFrmQcsLi2ceHx8/kJmjC63v5muZXwW8PiI2AS8Cvgd4H7AiIpaVvfpVwHSZPw2sBo5ExDJgOfCNuTeamTuBnQCjo6M5NjbWdrDJyUnGxsbYOkBfj3zLuhO85+D8/zkOXzfW2zAtmN3Gg8TMvTFomQctL3SfueNDPZn5jsxclZkjwDXAZzLzOuA+4A1l2hbgrrK8p1ymrP9Mdvp2Q5LUsdNxHv+twM0RMUXjGP6uMr4LuKCM3wxsPw33LUlaxCn5F7gycxKYLMtPAZfNM+ffgV84FfcnSeqcf7krSZWx+CWpMha/JFXG4pekylj8klQZi1+SKmPxS1JlLH5JqozFL0mVsfglqTIWvyRVxuKXpMpY/JJUGYtfkipj8UtSZSx+SaqMxS9JlbH4JakyFr8kVcbil6TKWPySVBmLX5IqY/FLUmUsfkmqjMUvSZXpuPgjYnVE3BcRj0bEIxFxUxk/PyL2RsQT5fd5ZTwi4v0RMRURD0fEpafqQUiSWresi+ueAG7JzAcj4qXAgYjYC2wF9mXmjojYDmwHbgWuBNaWn8uBD5bfWsTI9ntannt4x1WnMYmkM0HHe/yZeTQzHyzL/wY8BqwENgO7y7TdwNVleTPw4WzYD6yIiIs6Ti5J6khkZvc3EjECfBa4BPiXzFxRxgN4NjNXRMTdwI7M/FxZtw+4NTMfmHNb24BtAMPDw+snJibazjMzM8PQ0BAHp493/qB6bPjFcOz57m9n3crl3d9IC2a38SAxc28MWuZBywuLZx4fHz+QmaMLre/mUA8AETEE/C3wtsz8VqPrGzIzI6KtV5bM3AnsBBgdHc2xsbG2M01OTjI2NsbWNg6R9Nst607wnoNd/+fg8HVj3Ydpwew2HiRm7o1ByzxoeaH7zF2d1RMR302j9O/IzI+X4WOzh3DK76fL+DSwuunqq8qYJKmHujmrJ4BdwGOZ+UdNq/YAW8ryFuCupvE3l7N7rgCOZ+bRTu9fktSZbo4tvAp4E3AwIh4qY78B7AA+GhHXA18B3ljW3QtsAqaA54C3dHHfkqQOdVz85UPaWGD1hnnmJ3BDp/cnSTo1/MtdSaqMxS9JlbH4JakyFr8kVcbil6TKWPySVBmLX5IqY/FLUmUsfkmqjMUvSZWx+CWpMha/JFXG4pekylj8klQZi1+SKmPxS1JlLH5Jqkw3//SilqCR7fe0NO/wjqtOcxJJS5V7/JJUGYtfkipj8UtSZSx+SaqMxS9JlbH4Jakyns5ZKU/7lOrV8+KPiI3A+4CzgL/IzB29zqD+8QVH6r+eHuqJiLOADwBXAhcD10bExb3MIEm16/Ue/2XAVGY+BRARE8Bm4NEe51CLFtpDv2XdCba2uPd+OrX6DgJaz9zqu4127rsVvstRr0Rm9u7OIt4AbMzMXyqX3wRcnpk3Ns3ZBmwrF18BPN7BXV0IfL3LuL02aJkHLS+YuVcGLfOg5YXFM39fZr5soZVL7sPdzNwJ7OzmNiLigcwcPUWRemLQMg9aXjBzrwxa5kHLC91n7vXpnNPA6qbLq8qYJKlHel38nwfWRsSaiDgbuAbY0+MMklS1nh7qycwTEXEj8Ckap3PenpmPnIa76upQUZ8MWuZBywtm7pVByzxoeaHbw+G9/HBXktR/fmWDJFXG4pekypxRxR8RGyPi8YiYiojt/c4zn4hYHRH3RcSjEfFIRNxUxn87IqYj4qHys6nfWZtFxOGIOFiyPVDGzo+IvRHxRPl9Xr9zzoqIVzRty4ci4lsR8baltp0j4vaIeDoiDjWNzbtdo+H95fn9cERcukTy/kFEfLFk+kRErCjjIxHxfNO2/rNe5z1J5gWfBxHxjrKNH4+I1y6hzHc25T0cEQ+V8fa3c2aeET80Pix+Eng5cDbwBeDifueaJ+dFwKVl+aXAl2h8fcVvA7/e73wnyX0YuHDO2O8D28vyduDd/c55kufG14DvW2rbGXg1cClwaLHtCmwCPgkEcAVw/xLJ+xpgWVl+d1PekeZ5S2wbz/s8KP8vfgE4B1hTOuWspZB5zvr3AL/V6XY+k/b4//frIDLzO8Ds10EsKZl5NDMfLMv/BjwGrOxvqo5tBnaX5d3A1X3McjIbgCcz8yv9DjJXZn4WeGbO8ELbdTPw4WzYD6yIiIt6k7RhvryZ+enMPFEu7qfx9zlLxgLbeCGbgYnM/I/M/DIwRaNbeupkmSMigDcCH+n09s+k4l8JfLXp8hGWeKFGxAjwSuD+MnRjebt8+1I6bFIk8OmIOFC+VgNgODOPluWvAcP9ibaoa3jh/yRLeTvDwtt1EJ7jv0jjXcmsNRHxzxHxjxHxk/0KtYD5ngeDsI1/EjiWmU80jbW1nc+k4h8oETEE/C3wtsz8FvBB4PuBHwOO0ngrt5T8RGZeSuObVW+IiFc3r8zGe84ld25w+UPB1wN/U4aW+nZ+gaW6XecTEe8ETgB3lKGjwPdm5iuBm4G/jojv6Ve+OQbqeTDHtbxwR6bt7XwmFf/AfB1ERHw3jdK/IzM/DpCZxzLzvzLzv4E/pw9vL08mM6fL76eBT9DId2z2UEP5/XT/Ei7oSuDBzDwGS387Fwtt1yX7HI+IrcDrgOvKixXlcMk3yvIBGsfLf7BvIZuc5HmwZLcxQEQsA34OuHN2rJPtfCYV/0B8HUQ5PrcLeCwz/6hpvPlY7c8Ch+Zet18i4tyIeOnsMo0P8w7R2L5byrQtwF39SXhSL9g7WsrbuclC23UP8OZyds8VwPGmQ0J9E41/XOntwOsz87mm8ZdF49/gICJeDqwFnupPyhc6yfNgD3BNRJwTEWtoZP6nXuc7iZ8GvpiZR2YHOtrOvf60+jR/Er6JxlkyTwLv7HeeBTL+BI237g8DD5WfTcBfAgfL+B7gon5nbcr8chpnOnwBeGR22wIXAPuAJ4B/AM7vd9Y5uc8FvgEsbxpbUtuZxovSUeA/aRxPvn6h7UrjbJ4PlOf3QWB0ieSdonFcfPb5/Gdl7s+X58tDwIPAzyyhbbzg8wB4Z9nGjwNXLpXMZfxDwC/Pmdv2dvYrGySpMmfSoR5JUgssfkmqjMUvSZWx+CWpMha/JFXG4pekylj8klSZ/wEmDJk6BAzVDQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OXcswEIRPvGe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "max_seq_len = 25"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tk5S7DWaP2t6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# tokenize and encode sequences in the training set\n",
        "tokens_train = tokenizer.batch_encode_plus(\n",
        "    train_text.tolist(),\n",
        "    max_length = max_seq_len,\n",
        "    pad_to_max_length=True,\n",
        "    truncation=True,\n",
        "    return_token_type_ids=False\n",
        ")\n",
        "\n",
        "# tokenize and encode sequences in the validation set\n",
        "tokens_val = tokenizer.batch_encode_plus(\n",
        "    val_text.tolist(),\n",
        "    max_length = max_seq_len,\n",
        "    pad_to_max_length=True,\n",
        "    truncation=True,\n",
        "    return_token_type_ids=False\n",
        ")\n",
        "\n",
        "# tokenize and encode sequences in the test set\n",
        "tokens_test = tokenizer.batch_encode_plus(\n",
        "    test_text.tolist(),\n",
        "    max_length = max_seq_len,\n",
        "    pad_to_max_length=True,\n",
        "    truncation=True,\n",
        "    return_token_type_ids=False\n",
        ")"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wsm8bkRZQTw9",
        "colab_type": "text"
      },
      "source": [
        "### Convert Integer Sequences to Tensors"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QR-lXwmzQPd6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# for train set\n",
        "train_seq = torch.tensor(tokens_train['input_ids'])\n",
        "train_mask = torch.tensor(tokens_train['attention_mask'])\n",
        "train_y = torch.tensor(train_labels.tolist())\n",
        "\n",
        "# for validation set\n",
        "val_seq = torch.tensor(tokens_val['input_ids'])\n",
        "val_mask = torch.tensor(tokens_val['attention_mask'])\n",
        "val_y = torch.tensor(val_labels.tolist())\n",
        "\n",
        "# for test set\n",
        "test_seq = torch.tensor(tokens_test['input_ids'])\n",
        "test_mask = torch.tensor(tokens_test['attention_mask'])\n",
        "test_y = torch.tensor(test_labels.tolist())"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ov1cOBlcRLuk",
        "colab_type": "text"
      },
      "source": [
        "## Create DataLoaders"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qUy9JKFYQYLp",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "\n",
        "#define a batch size\n",
        "batch_size = 32\n",
        "\n",
        "# wrap tensors\n",
        "train_data = TensorDataset(train_seq, train_mask, train_y)\n",
        "\n",
        "# sampler for sampling the data during training\n",
        "train_sampler = RandomSampler(train_data)\n",
        "\n",
        "# dataLoader for train set\n",
        "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
        "\n",
        "# wrap tensors\n",
        "val_data = TensorDataset(val_seq, val_mask, val_y)\n",
        "\n",
        "# sampler for sampling the data during training\n",
        "val_sampler = SequentialSampler(val_data)\n",
        "\n",
        "# dataLoader for validation set\n",
        "val_dataloader = DataLoader(val_data, sampler = val_sampler, batch_size=batch_size)"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K2HZc5ZYRV28",
        "colab_type": "text"
      },
      "source": [
        "## Freeze BERT Parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wHZ0MC00RQA_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# freeze all the parameters\n",
        "for param in bert.parameters():\n",
        "    param.requires_grad = False"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s7ahGBUWRi3X",
        "colab_type": "text"
      },
      "source": [
        "## Define Model Architecture"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b3iEtGyYRd0A",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class BERT_Arch(nn.Module):\n",
        "\n",
        "    def __init__(self, bert):\n",
        "      \n",
        "      super(BERT_Arch, self).__init__()\n",
        "\n",
        "      self.bert = bert \n",
        "      \n",
        "      # dropout layer\n",
        "      self.dropout = nn.Dropout(0.1)\n",
        "      \n",
        "      # relu activation function\n",
        "      self.relu =  nn.ReLU()\n",
        "\n",
        "      # dense layer 1\n",
        "      self.fc1 = nn.Linear(768,512)\n",
        "      \n",
        "      # dense layer 2 (Output layer)\n",
        "      self.fc2 = nn.Linear(512,2)\n",
        "\n",
        "      #softmax activation function\n",
        "      self.softmax = nn.LogSoftmax(dim=1)\n",
        "\n",
        "    #define the forward pass\n",
        "    def forward(self, sent_id, mask):\n",
        "\n",
        "      #pass the inputs to the model  \n",
        "      _, cls_hs = self.bert(sent_id, attention_mask=mask)\n",
        "      \n",
        "      x = self.fc1(cls_hs)\n",
        "\n",
        "      x = self.relu(x)\n",
        "\n",
        "      x = self.dropout(x)\n",
        "\n",
        "      # output layer\n",
        "      x = self.fc2(x)\n",
        "      \n",
        "      # apply softmax activation\n",
        "      x = self.softmax(x)\n",
        "\n",
        "      return x"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cBAJJVuJRliv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# pass the pre-trained BERT to our define architecture\n",
        "model = BERT_Arch(bert)\n",
        "\n",
        "# push the model to GPU\n",
        "model = model.to(device)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "taXS0IilRn9J",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# optimizer from hugging face transformers\n",
        "from transformers import AdamW\n",
        "\n",
        "# define the optimizer\n",
        "optimizer = AdamW(model.parameters(), lr = 1e-3)"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j9CDpoMQR_rK",
        "colab_type": "text"
      },
      "source": [
        "## Find Class Weights"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "izY5xH5eR7Ur",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "31ec23c2-06a1-4fbf-e9ab-080b0d3fd89b"
      },
      "source": [
        "from sklearn.utils.class_weight import compute_class_weight\n",
        "\n",
        "#compute the class weights\n",
        "class_wts = compute_class_weight('balanced', np.unique(train_labels), train_labels)\n",
        "\n",
        "print(class_wts)"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[0.57743559 3.72848948]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r1WvfY2vSGKi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# convert class weights to tensor\n",
        "weights= torch.tensor(class_wts,dtype=torch.float)\n",
        "weights = weights.to(device)\n",
        "\n",
        "# loss function\n",
        "cross_entropy  = nn.NLLLoss(weight=weights) \n",
        "\n",
        "# number of training epochs\n",
        "epochs = 10"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "My4CA0qaShLq",
        "colab_type": "text"
      },
      "source": [
        "## Fine-Tune BERT"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rskLk8R_SahS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# function to train the model\n",
        "def train():\n",
        "  \n",
        "  model.train()\n",
        "\n",
        "  total_loss, total_accuracy = 0, 0\n",
        "  \n",
        "  # empty list to save model predictions\n",
        "  total_preds=[]\n",
        "  \n",
        "  # iterate over batches\n",
        "  for step,batch in enumerate(train_dataloader):\n",
        "    \n",
        "    # progress update after every 50 batches.\n",
        "    if step % 50 == 0 and not step == 0:\n",
        "      print('  Batch {:>5,}  of  {:>5,}.'.format(step, len(train_dataloader)))\n",
        "\n",
        "    # push the batch to gpu\n",
        "    batch = [r.to(device) for r in batch]\n",
        " \n",
        "    sent_id, mask, labels = batch\n",
        "\n",
        "    # clear previously calculated gradients \n",
        "    model.zero_grad()        \n",
        "\n",
        "    # get model predictions for the current batch\n",
        "    preds = model(sent_id, mask)\n",
        "\n",
        "    # compute the loss between actual and predicted values\n",
        "    loss = cross_entropy(preds, labels)\n",
        "\n",
        "    # add on to the total loss\n",
        "    total_loss = total_loss + loss.item()\n",
        "\n",
        "    # backward pass to calculate the gradients\n",
        "    loss.backward()\n",
        "\n",
        "    # clip the the gradients to 1.0. It helps in preventing the exploding gradient problem\n",
        "    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "    # update parameters\n",
        "    optimizer.step()\n",
        "\n",
        "    # model predictions are stored on GPU. So, push it to CPU\n",
        "    preds=preds.detach().cpu().numpy()\n",
        "\n",
        "    # append the model predictions\n",
        "    total_preds.append(preds)\n",
        "\n",
        "  # compute the training loss of the epoch\n",
        "  avg_loss = total_loss / len(train_dataloader)\n",
        "  \n",
        "  # predictions are in the form of (no. of batches, size of batch, no. of classes).\n",
        "  # reshape the predictions in form of (number of samples, no. of classes)\n",
        "  total_preds  = np.concatenate(total_preds, axis=0)\n",
        "\n",
        "  #returns the loss and predictions\n",
        "  return avg_loss, total_preds"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yGXovFDlSxB5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# function for evaluating the model\n",
        "def evaluate():\n",
        "  \n",
        "  print(\"\\nEvaluating...\")\n",
        "  \n",
        "  # deactivate dropout layers\n",
        "  model.eval()\n",
        "\n",
        "  total_loss, total_accuracy = 0, 0\n",
        "  \n",
        "  # empty list to save the model predictions\n",
        "  total_preds = []\n",
        "\n",
        "  # iterate over batches\n",
        "  for step,batch in enumerate(val_dataloader):\n",
        "    \n",
        "    # Progress update every 50 batches.\n",
        "    if step % 50 == 0 and not step == 0:\n",
        "      \n",
        "      # Calculate elapsed time in minutes.\n",
        "      elapsed = format_time(time.time() - t0)\n",
        "            \n",
        "      # Report progress.\n",
        "      print('  Batch {:>5,}  of  {:>5,}.'.format(step, len(val_dataloader)))\n",
        "\n",
        "    # push the batch to gpu\n",
        "    batch = [t.to(device) for t in batch]\n",
        "\n",
        "    sent_id, mask, labels = batch\n",
        "\n",
        "    # deactivate autograd\n",
        "    with torch.no_grad():\n",
        "      \n",
        "      # model predictions\n",
        "      preds = model(sent_id, mask)\n",
        "\n",
        "      # compute the validation loss between actual and predicted values\n",
        "      loss = cross_entropy(preds,labels)\n",
        "\n",
        "      total_loss = total_loss + loss.item()\n",
        "\n",
        "      preds = preds.detach().cpu().numpy()\n",
        "\n",
        "      total_preds.append(preds)\n",
        "\n",
        "  # compute the validation loss of the epoch\n",
        "  avg_loss = total_loss / len(val_dataloader) \n",
        "\n",
        "  # reshape the predictions in form of (number of samples, no. of classes)\n",
        "  total_preds  = np.concatenate(total_preds, axis=0)\n",
        "\n",
        "  return avg_loss, total_preds"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9KZEgxRRTLXG",
        "colab_type": "text"
      },
      "source": [
        "## Model Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k1USGTntS3TS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "9c6f6c28-8fe6-4b39-8864-e61c443cded1"
      },
      "source": [
        "# set initial loss to infinite\n",
        "best_valid_loss = float('inf')\n",
        "\n",
        "# empty lists to store training and validation loss of each epoch\n",
        "train_losses=[]\n",
        "valid_losses=[]\n",
        "\n",
        "#for each epoch\n",
        "for epoch in range(epochs):\n",
        "     \n",
        "    print('\\n Epoch {:} / {:}'.format(epoch + 1, epochs))\n",
        "    \n",
        "    #train model\n",
        "    train_loss, _ = train()\n",
        "    \n",
        "    #evaluate model\n",
        "    valid_loss, _ = evaluate()\n",
        "    \n",
        "    #save the best model\n",
        "    if valid_loss < best_valid_loss:\n",
        "        best_valid_loss = valid_loss\n",
        "        torch.save(model.state_dict(), 'saved_weights.pt')\n",
        "    \n",
        "    # append training and validation loss\n",
        "    train_losses.append(train_loss)\n",
        "    valid_losses.append(valid_loss)\n",
        "    \n",
        "    print(f'\\nTraining Loss: {train_loss:.3f}')\n",
        "    print(f'Validation Loss: {valid_loss:.3f}')"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            " Epoch 1 / 10\n",
            "  Batch    50  of    122.\n",
            "  Batch   100  of    122.\n",
            "\n",
            "Evaluating...\n",
            "\n",
            "Training Loss: 0.473\n",
            "Validation Loss: 0.239\n",
            "\n",
            " Epoch 2 / 10\n",
            "  Batch    50  of    122.\n",
            "  Batch   100  of    122.\n",
            "\n",
            "Evaluating...\n",
            "\n",
            "Training Loss: 0.331\n",
            "Validation Loss: 0.208\n",
            "\n",
            " Epoch 3 / 10\n",
            "  Batch    50  of    122.\n",
            "  Batch   100  of    122.\n",
            "\n",
            "Evaluating...\n",
            "\n",
            "Training Loss: 0.274\n",
            "Validation Loss: 0.178\n",
            "\n",
            " Epoch 4 / 10\n",
            "  Batch    50  of    122.\n",
            "  Batch   100  of    122.\n",
            "\n",
            "Evaluating...\n",
            "\n",
            "Training Loss: 0.278\n",
            "Validation Loss: 0.255\n",
            "\n",
            " Epoch 5 / 10\n",
            "  Batch    50  of    122.\n",
            "  Batch   100  of    122.\n",
            "\n",
            "Evaluating...\n",
            "\n",
            "Training Loss: 0.293\n",
            "Validation Loss: 0.154\n",
            "\n",
            " Epoch 6 / 10\n",
            "  Batch    50  of    122.\n",
            "  Batch   100  of    122.\n",
            "\n",
            "Evaluating...\n",
            "\n",
            "Training Loss: 0.202\n",
            "Validation Loss: 0.140\n",
            "\n",
            " Epoch 7 / 10\n",
            "  Batch    50  of    122.\n",
            "  Batch   100  of    122.\n",
            "\n",
            "Evaluating...\n",
            "\n",
            "Training Loss: 0.248\n",
            "Validation Loss: 0.146\n",
            "\n",
            " Epoch 8 / 10\n",
            "  Batch    50  of    122.\n",
            "  Batch   100  of    122.\n",
            "\n",
            "Evaluating...\n",
            "\n",
            "Training Loss: 0.183\n",
            "Validation Loss: 0.311\n",
            "\n",
            " Epoch 9 / 10\n",
            "  Batch    50  of    122.\n",
            "  Batch   100  of    122.\n",
            "\n",
            "Evaluating...\n",
            "\n",
            "Training Loss: 0.189\n",
            "Validation Loss: 0.133\n",
            "\n",
            " Epoch 10 / 10\n",
            "  Batch    50  of    122.\n",
            "  Batch   100  of    122.\n",
            "\n",
            "Evaluating...\n",
            "\n",
            "Training Loss: 0.241\n",
            "Validation Loss: 0.322\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_yrhUc9kTI5a",
        "colab_type": "text"
      },
      "source": [
        "## Load Saved Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OacxUyizS8d1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "ab9b49cb-686d-4fdb-c519-b3d407e69b41"
      },
      "source": [
        "#load weights of best model\n",
        "path = 'saved_weights.pt'\n",
        "model.load_state_dict(torch.load(path))"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x4SVftkkTZXA",
        "colab_type": "text"
      },
      "source": [
        "## Get Predictions for Test Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NZl0SZmFTRQA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# get predictions for test data\n",
        "with torch.no_grad():\n",
        "  preds = model(test_seq.to(device), test_mask.to(device))\n",
        "  preds = preds.detach().cpu().numpy()"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ms1ObHZxTYSI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        },
        "outputId": "27d9147b-c40f-4939-ff2e-641528c69bc9"
      },
      "source": [
        "# model's performance\n",
        "preds = np.argmax(preds, axis = 1)\n",
        "print(classification_report(test_y, preds))"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.99      0.98      0.98       724\n",
            "           1       0.85      0.95      0.90       112\n",
            "\n",
            "    accuracy                           0.97       836\n",
            "   macro avg       0.92      0.96      0.94       836\n",
            "weighted avg       0.97      0.97      0.97       836\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YqzLS7rHTp4T",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 142
        },
        "outputId": "e14e01f0-70d5-4400-8939-b9b06c655212"
      },
      "source": [
        "# confusion matrix\n",
        "pd.crosstab(test_y, preds)"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th>col_0</th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>row_0</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>706</td>\n",
              "      <td>18</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>6</td>\n",
              "      <td>106</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "col_0    0    1\n",
              "row_0          \n",
              "0      706   18\n",
              "1        6  106"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    }
  ]
}