Transformer_Experiments.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.2"
    },
    "colab": {
      "name": "Transformer_Experiments.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "keCOzE-k4F9F",
        "colab_type": "text"
      },
      "source": [
        "# Transformer Language Model Experiments\n",
        "\n",
        "## ML Classification for Records Management\n",
        "\n",
        "Jason Franks\n",
        "\n",
        "Master of Data Science Minor Thesis\n",
        "\n",
        "Supervisors: Dr Greg Rolan, Dr Lan Du\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j5Vhvfh54WIS",
        "colab_type": "text"
      },
      "source": [
        "## Install CUDA and SimpleTransformers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ckt45UQ2oTKS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "%%writefile setup.sh\n",
        "export CUDA_HOME=/usr/local/cuda-10.1\n",
        "git clone https://github.com/NVIDIA/apex\n",
        "pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ./apex"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vmspaDV9pSA6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!sh setup.sh"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rxd2QUkGsEpI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!pip install simpletransformers"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xDe0TpZInR7S",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import sys\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import confusion_matrix, precision_score, recall_score, accuracy_score, f1_score, matthews_corrcoef, make_scorer, balanced_accuracy_score\n",
        "from simpletransformers.classification import ClassificationModel\n",
        "\n",
        "from functools import partial\n",
        "import io\n",
        "\n",
        "import nltk as nltk\n",
        "from nltk.collocations import BigramCollocationFinder, BigramAssocMeasures, TrigramCollocationFinder, TrigramAssocMeasures\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.tokenize import RegexpTokenizer\n",
        "from nltk.tokenize.mwe import MWETokenizer\n",
        "\n",
        "from google.colab import files\n",
        "from google.colab import drive"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7rL0oZUbuv1E",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VTryqGq94hJL",
        "colab_type": "text"
      },
      "source": [
        "## **Set the following variables to load the data**\n",
        "\n",
        "**mount_path**: path into a google drive to your working folder\n",
        "\n",
        "**data_file**: name of the file containing your data. This must be a tab-separated .tsv file with two columns: 'label', containing the category name, and 'text', containing the record's raw text.\n",
        "\n",
        "**model_type**: Select the language model to train fro ['xlnet', 'bert', 'roberta']\n",
        "\n",
        "Evey category in the data file should have *at least* 10 records.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hPBwavz3u5Li",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mount_path = '/content/drive/My Drive/'\n",
        "data_file = 'all_docs_trimmed.tsv'\n",
        "model_type = 'xlnet'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AsnaqUeo416w",
        "colab_type": "text"
      },
      "source": [
        "## Import and prepare the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FOsUXzH8vArz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "all_docs = pd.read_csv(mount_path + data_file, \"\\t\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CtPZh_UenR7c",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "labels = all_docs['label'].unique()\n",
        "num_labels = len(labels)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P3ilTuP4c5IV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Utility functions to help assess the output\n",
        "\n",
        "def get_within_category_accuracies( cat_list, cm ):\n",
        "    cat_accuracies = []\n",
        "    for row in range(len(cat_list)):\n",
        "        cm_row = cm[row]\n",
        "        num_correct = cm_row[row]\n",
        "\n",
        "        total = sum(cm[row])\n",
        "        if total == 0:\n",
        "            continue\n",
        "        \n",
        "        cat_accuracies.append(num_correct/total)\n",
        "            \n",
        "    df = pd.DataFrame(zip(cat_list, cat_accuracies), columns=['label', 'accuracy'])\n",
        "    return df\n",
        "\n",
        "def assess_model(test, preds, title, labels, draw_plot=True):        \n",
        "    final_test_accuracy = accuracy_score(test, preds)\n",
        "    final_test_f1 = f1_score(test, preds, average='macro') \n",
        "    final_cat_f1s = f1_score(test, preds, average=None) \n",
        "    final_test_f1_weighted = f1_score(test, preds, average='weighted')    \n",
        "    final_test_precision = precision_score(test, preds, average='macro') \n",
        "    final_cat_precision = precision_score(test, preds, average=None) \n",
        "    final_test_precision_weighted = precision_score(test, preds, average='weighted')    \n",
        "    final_test_recall = recall_score(test, preds, average='macro') \n",
        "    final_cat_recall = recall_score(test, preds, average=None) \n",
        "    final_test_recall_weighted = recall_score(test, preds, average='weighted')    \n",
        "    cm = confusion_matrix(test, preds)\n",
        "\n",
        "    metrics=[]\n",
        "    metrics.append( [\"accuracy\", final_test_accuracy])\n",
        "    metrics.append( [\"f1\", final_test_f1])\n",
        "    metrics.append( [\"f1 weighted\", final_test_f1_weighted])\n",
        "    metrics.append( [\"precision\", final_test_precision])\n",
        "    metrics.append( [\"precision weighted\", final_test_precision_weighted])\n",
        "    metrics.append( [\"recall\", final_test_recall])\n",
        "    metrics.append( [\"recall weighted\", final_test_recall_weighted])\n",
        "\n",
        "    print( \"------------Model assessment-----\")\n",
        "\n",
        "    print( \"test f1 / category, {}\\n\".format( final_cat_f1s))   \n",
        "    print( \"test precision / category, {}\\n\".format( final_cat_precision))   \n",
        "    print( \"test recall / category, {}\\n\".format( final_cat_recall))   \n",
        "    \n",
        "    model_assessment = pd.DataFrame(metrics, columns=[\"metric\", \"value\"])\n",
        "    print(model_assessment)\n",
        "    model_assessment.to_csv(f'{mount_path}/{title}_assess.csv', index=False )\n",
        "\n",
        "    acc_by_cat = get_within_category_accuracies( labels, cm)\n",
        "\n",
        "    acc_by_cat.to_csv(f'{mount_path}/output/{title}_acc_by_cat.csv', index=False )\n",
        "\n",
        "    if draw_plot:\n",
        "      ax = acc_by_cat.plot.bar( x='label', y='accuracy', title=f'{title} Accuracy by Category', legend=None, figsize=(20,20), fontsize=14)\n",
        "      ax.set_ylabel(\"Accuracy\", fontsize=12)\n",
        "      ax.set_xticklabels(labels, rotation=90, fontsize=12)\n",
        "      plt.tight_layout()\n",
        "      fig = ax.get_figure()\n",
        "      fig.savefig(mount_path + f'/output/{title}_Accuracy_by_Category.png', dpi=300)\n",
        "    \n",
        "    print(\"-------Confusion Matrix---------\")\n",
        "    print(cm)\n",
        "    \n",
        "    cmDF = pd.DataFrame.from_records(cm)    \n",
        "    cmDF.columns=labels\n",
        "    cmDF.index=labels\n",
        "    cmDF.to_csv(f'{mount_path}/output/{title}_cm.csv', index=True)\n",
        "\n",
        "    return acc_by_cat\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5ZoUMB9wnR7h",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "all_docs['label'] = all_docs['label'].astype('category').cat.codes"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zITJy2TZdSyF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def isNumber(s):    \n",
        "      try:\n",
        "          float(s)\n",
        "          return True\n",
        "      except ValueError:\n",
        "          return False\n",
        "\n",
        "def hasNumbers(inputString):\n",
        "    return any(char.isdigit() for char in inputString)\n",
        "\n",
        "# Clean text and vectorize\n",
        "def clean_and_drop_stopwords( df, lowercase = False ):\n",
        "  tokenizer = RegexpTokenizer(r\"\\w+(?:[-.]\\w+)?\")\n",
        "  if lowercase:\n",
        "    df['text'] = df['text'].str.lower() \n",
        "\n",
        "  df['pretext'] = df['text'].apply(lambda x: tokenizer.tokenize(x))\n",
        "\n",
        "  \n",
        "  nltk.download('stopwords')\n",
        "  stopWords = set(stopwords.words('english'))\n",
        "  df['posttext'] = df['pretext'].apply(lambda toks: [word for word in toks if not word in stopWords])\n",
        "\n",
        "  df['posttext'] = df['posttext'].apply(lambda toks: [word for word in toks if not hasNumbers(word)])\n",
        "\n",
        "  df['posttext'] = df['posttext'].apply(lambda toks: [word for word in toks if len(word) > 2])\n",
        "\n",
        "  df['text'] = df['posttext'].apply(lambda x: ' '.join(x))  \n",
        "  df.drop( ['posttext', 'pretext'], axis=1)\n",
        "  return df"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YmU2Axe9dbzO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "lowercase = ( model_name == 'roberta')\n",
        "\n",
        "df = clean_and_drop_stopwords(all_docs, lowercase)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EOG8-uYHnR71",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = df.reset_index()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vAfpbtUunR78",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df['posttext'] = df['posttext'].apply(lambda x: ' '.join(x))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bU5s6dIonR7_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df['text'] = df['posttext']\n",
        "df.drop('pretext', axis=1, inplace=True)\n",
        "df.drop('index', axis=1, inplace=True)\n",
        "df.drop('posttext', axis=1, inplace=True)\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xelacy3J9sFt",
        "colab_type": "text"
      },
      "source": [
        "## Split test and train sets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yUDmtbcpnR8D",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train, test = train_test_split(df, test_size=0.2, random_state=94606619, stratify=df[['label']])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ko7l2y2QnR8K",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "del all_docs"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2pF2m-aZ9ztM",
        "colab_type": "text"
      },
      "source": [
        "## Train model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6pBq8rNbnR8N",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "if model_type == 'xlnet':\n",
        "  model_name = 'xlnet-base-cased' \n",
        "elif model_type == 'bert':\n",
        "  model_name = 'bert-base-cased'\n",
        "elif model_type == 'roberta':\n",
        "  model_name = 'roberta-base'\n",
        "\n",
        "\n",
        "model = ClassificationModel(model_type, model_name, num_labels=num_labels,  args={ \"num_train_epochs\": 18, \"save_eval_checkpoints\" : False,  \"save_model_every_epoch\": False, \"sliding_window\": False, 'overwrite_output_dir': True, \"max_seq_length\": 256}, use_cuda=True)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f9TjHemenR8O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Train the model\n",
        "model.train_model(train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8Gi2pX349466",
        "colab_type": "text"
      },
      "source": [
        "## Assess Results"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dWA1vo6yBl7C",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def wrap_f1( preds, y):\n",
        "  f1s = f1_score(preds, y, average='macro')\n",
        "  print(f1s)\n",
        "  return np.mean(f1s)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Msud2sbqnR8Q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "result, model_outputs, wrong_predictions = model.eval_model(test, acc=accuracy_score, balanced_accuracy=balanced_accuracy_score, f1=wrap_f1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9k-lYPs1nR8U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "result"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sJRVV7jAnR8T",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "accuracy = 1- len(wrong_predictions)/len(test)\n",
        "accuracy\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uUk0hMTdm0qy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "preds = model.predict( test['text'].tolist())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z88NsC57ApL8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "acc_by_cat = assess_model(test['label'], preds[0], model_type, labels)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}