SAE_error/hotpot_qa_ai_analysis.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UP1CACu3irWL"
      },
      "source": [
        "# Imports"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lzsjluAjzb3C",
        "outputId": "273533f0-9edb-4868-cf3b-7098526c4958"
      },
      "outputs": [],
      "source": [
        "!pip install transformers\n",
        "!pip install datasets\n",
        "!pip install sentencepiece\n",
        "!pip install scikit-learn-extra"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JRNFoaJXvO4x",
        "outputId": "4d1162ca-cad2-42ad-efa1-ea1c601bb513"
      },
      "outputs": [],
      "source": [
        "import math\n",
        "import torch\n",
        "import numpy as np\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import argparse\n",
        "from __future__ import print_function\n",
        "from collections import Counter\n",
        "import string\n",
        "import re\n",
        "import argparse\n",
        "import json\n",
        "import os\n",
        "import random\n",
        "import shutil\n",
        "import time\n",
        "import torch.nn.parallel\n",
        "import torch.backends.cudnn as cudnn\n",
        "from tqdm import tqdm\n",
        "import torch.optim\n",
        "import torch.utils.data\n",
        "import torchvision.transforms as transforms\n",
        "import torchvision.datasets as datasets\n",
        "from torch.autograd import Variable\n",
        "from PIL import Image\n",
        "import torch.utils.data as data\n",
        "from torchvision.datasets.utils import download_url, check_integrity\n",
        "import sys\n",
        "from sklearn.decomposition import PCA\n",
        "from matplotlib import pyplot as plt\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "import pickle\n",
        "from sklearn.gaussian_process.kernels import RBF\n",
        "import torch\n",
        "from scipy.stats import multivariate_normal\n",
        "import  scipy.stats as st\n",
        "from matplotlib import cm\n",
        "import torch.optim as optim\n",
        "from __future__ import print_function\n",
        "from transformers import AutoTokenizer, AutoModelForQuestionAnswering, AdamW, AutoModel\n",
        "import nltk\n",
        "from torch.utils.data import DataLoader\n",
        "import pickle\n",
        "import spacy\n",
        "from gensim import corpora, models, similarities\n",
        "import gensim\n",
        "from spacy.lang.en import English\n",
        "from nltk.corpus import wordnet as wn\n",
        "from nltk.stem.wordnet import WordNetLemmatizer\n",
        "\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "print(device)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5WSnSiY1dF2E"
      },
      "source": [
        "# HotpotQA\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "e-1r7KAwnJZJ"
      },
      "outputs": [],
      "source": [
        "DISTRACTORS_PARS_LEN = 0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3E0eakH8dHfv"
      },
      "outputs": [],
      "source": [
        "from datasets import load_dataset\n",
        "\n",
        "dataset = load_dataset(\"hotpot_qa\", 'distractor')\n",
        "train_dataset = dataset['train']\n",
        "validation_dataset = dataset['validation']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bqzhee23lkrz"
      },
      "outputs": [],
      "source": [
        "\n",
        "def get_token_lenght(context, question):\n",
        "    inputs = tokenizer.encode_plus(question, context, add_special_tokens=True, return_tensors=\"pt\")\n",
        "    return len(inputs['input_ids'][0])\n",
        "\n",
        "def remove_distractors(example):\n",
        "    all_pars = example['context']['title']\n",
        "    gold_pars = list(set(example['supporting_facts']['title']))\n",
        "    distractor_pars = list(set(all_pars) - set(gold_pars))\n",
        "    # get indices to keep from disractors\n",
        "    if len(distractor_pars) == 0 or DISTRACTORS_PARS_LEN == 0:\n",
        "        distract_indices = []\n",
        "    else:\n",
        "        distract_indices = random.sample(range(len(distractor_pars)), DISTRACTORS_PARS_LEN)\n",
        "    distractor_pars = [distractor_pars[idx] for idx in distract_indices]\n",
        "    keep_pars = gold_pars + distractor_pars\n",
        "    keep_pars_indices = [all_pars.index(keep_par) for keep_par in keep_pars]\n",
        "    example['context']['title'] = [example['context']['title'][idx] for idx in keep_pars_indices]\n",
        "    example['context']['sentences'] = [\"\".join(example['context']['sentences'][idx]) for idx in keep_pars_indices]\n",
        "    example['passage'] = \"\".join(example['context']['sentences'])\n",
        "    return example"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 114,
          "referenced_widgets": [
            "5e623b98871a4d9298d7c51908f7214d",
            "16427970de0f4bf1993057a03ca794f8",
            "16d94096022c4dea98a298ef76ecc634",
            "c906e0e8c43d4a09b948eef339ca0a09",
            "558617f2b4674f08bc1eccb0943e5bde",
            "c5f078d33bc0446eb28a8e9115e5f4fe",
            "9d988b67ff1343989cab780a1f3e6c87",
            "9c3d199fa1f64dd4a21c3d7bceed52f3",
            "92aa6afcfe1247398e49888b26a6ecfb",
            "a2f97d589036400494cc7d8c4fa4727d",
            "437c83dfbf4a48b4a6ac54763cbef34e",
            "60ebf29513bc4fad96ff1dbc5a8f289b",
            "f99a63e054634f4cb62a9fb6a42d2cb7",
            "93753133e53740fab9d14afc1fafc15e",
            "d8e89b135b16432599ff73b8c9694bcb",
            "f7c255b791304f7cb95cb8aba66e179c"
          ]
        },
        "id": "unz9KJ9qofO3",
        "outputId": "345414a9-a00b-459e-ea85-f6559048a527"
      },
      "outputs": [],
      "source": [
        "train_dataset = train_dataset.map(remove_distractors)\n",
        "validation_dataset = validation_dataset.map(remove_distractors)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GIh1WdiuzOeV"
      },
      "outputs": [],
      "source": [
        "START_TEST = math.floor(len(validation_dataset)/2)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LChYrx5dAe--",
        "jupyter": {
          "source_hidden": true
        },
        "outputId": "8e57cde1-ca20-4b29-820c-8d86b5754082"
      },
      "outputs": [],
      "source": [
        "def read_hotpot(hotpot_dataset):\n",
        "\n",
        "    contexts = []\n",
        "    questions = []\n",
        "    answers = []\n",
        "    all_contexts = hotpot_dataset['context'][:]\n",
        "    all_questions = hotpot_dataset['question'][:]\n",
        "    all_answers = hotpot_dataset['answer'][:]\n",
        "    invalid_quests = 0\n",
        "    for i in range(len(all_contexts)):\n",
        "\n",
        "        answer = {}\n",
        "        context = \"\".join(all_contexts[i]['sentences'])\n",
        "        answer['text'] = all_answers[i]\n",
        "        answer_start = context.find(answer['text'])\n",
        "        if answer_start == -1:\n",
        "            if answer['text'] not in [\"yes\", \"no\"]:\n",
        "                invalid_quests += 1\n",
        "            answer['answer_start'] = answer_start\n",
        "            answers.append(answer)\n",
        "            contexts.append(context)\n",
        "            questions.append(all_questions[i])\n",
        "        else:\n",
        "            answer['answer_start'] = answer_start\n",
        "            answers.append(answer)\n",
        "            contexts.append(context)\n",
        "            questions.append(all_questions[i])\n",
        "    print(f'invalid percentage {invalid_quests/len(hotpot_dataset)*100:.2f}')\n",
        "    return contexts, questions, answers\n",
        "'''\n",
        "START_TEST = math.floor(len(validation_dataset)/2)\n",
        "val_contexts, val_questions, val_answers = read_hotpot(validation_dataset)\n",
        "teaching_contexts = val_contexts[:START_TEST]\n",
        "teaching_questions = val_questions[:START_TEST]\n",
        "teaching_answers = val_answers[:START_TEST]\n",
        "validation_contexts = val_contexts[START_TEST:]\n",
        "validation_questions = val_questions[START_TEST:]\n",
        "validation_answers = val_answers[START_TEST:]\n",
        "'''"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AK9JLhs-97wO"
      },
      "source": [
        "## Utils"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "40Mqih9Z8rTx"
      },
      "outputs": [],
      "source": [
        "\n",
        "def normalize_answer(s):\n",
        "    \"\"\"Lower text and remove punctuation, articles and extra whitespace.\"\"\"\n",
        "    def remove_articles(text):\n",
        "        return re.sub(r'\\b(a|an|the)\\b', ' ', text)\n",
        "\n",
        "    def white_space_fix(text):\n",
        "        return ' '.join(text.split())\n",
        "\n",
        "    def remove_punc(text):\n",
        "        exclude = set(string.punctuation)\n",
        "        return ''.join(ch for ch in text if ch not in exclude)\n",
        "\n",
        "    def lower(text):\n",
        "        return text.lower()\n",
        "\n",
        "    return white_space_fix(remove_articles(remove_punc(lower(s))))\n",
        "\n",
        "\n",
        "def f1_score(prediction, ground_truth):\n",
        "    prediction_tokens = normalize_answer(prediction).split()\n",
        "    ground_truth_tokens = normalize_answer(ground_truth).split()\n",
        "    common = Counter(prediction_tokens) & Counter(ground_truth_tokens)\n",
        "    num_same = sum(common.values())\n",
        "    if num_same == 0:\n",
        "        return 0\n",
        "    precision = 1.0 * num_same / len(prediction_tokens)\n",
        "    recall = 1.0 * num_same / len(ground_truth_tokens)\n",
        "    f1 = (2 * precision * recall) / (precision + recall)\n",
        "    return f1\n",
        "\n",
        "\n",
        "def exact_match_score(prediction, ground_truth):\n",
        "    return (normalize_answer(prediction) == normalize_answer(ground_truth))\n",
        "\n",
        "\n",
        "def metric_max_over_ground_truths(metric_fn, prediction, ground_truths):\n",
        "    scores_for_ground_truths = []\n",
        "    for ground_truth in ground_truths:\n",
        "        score = metric_fn(prediction, ground_truth)\n",
        "        scores_for_ground_truths.append(score)\n",
        "    return max(scores_for_ground_truths)\n",
        "\n",
        "\n",
        "\n",
        "def evaluate_truth_pred(truths, preds):\n",
        "    '''\n",
        "    truths, preds: matched arrays of ground truth answers and predictions\n",
        "    '''\n",
        "    f1 = exact_match = total = 0\n",
        "    for i in range(len(truths)):\n",
        "        total += 1\n",
        "        if truths[i] in ['yes', \"no\"]:\n",
        "            continue\n",
        "        ground_truths = [truths[i]]\n",
        "        prediction = preds[i]\n",
        "        exact_match += metric_max_over_ground_truths(\n",
        "            exact_match_score, prediction, ground_truths)\n",
        "        f1 += metric_max_over_ground_truths(\n",
        "            f1_score, prediction, ground_truths)\n",
        "\n",
        "    exact_match = 100.0 * exact_match / (total+ 0.00000000001)\n",
        "    f1 = 100.0 * f1 / (total+ 0.00000000001)\n",
        "\n",
        "    return {'exact_match': exact_match, 'f1': f1}\n",
        "\n",
        "\n",
        "def get_answer( model, tokenizer, context, question):\n",
        "    # 1. TOKENIZE THE INPUT\n",
        "    # note: if you don't include return_tensors='pt' you'll get a list of lists which is easier for\n",
        "    # exploration but you cannot feed that into a model.\n",
        "    inputs = tokenizer.encode_plus(question, context, return_tensors=\"pt\")\n",
        "    inputs = inputs.to(device)\n",
        "    # 2. OBTAIN MODEL SCORES\n",
        "    # the AutoModelForQuestionAnswering class includes a span predictor on top of the model.\n",
        "    # the model returns answer start and end scores for each word in the text\n",
        "    answer_start_scores, answer_end_scores = model(**inputs, return_dict=False)\n",
        "    answer_start = torch.argmax(answer_start_scores)  # get the most likely beginning of answer with the argmax of the score\n",
        "    answer_end = torch.argmax(answer_end_scores) + 1  # get the most likely end of answer with the argmax of the score\n",
        "    # 3. GET THE ANSWER SPAN\n",
        "    # once we have the most likely start and end tokens, we grab all the tokens between them\n",
        "    # and convert tokens back to words!\n",
        "    answer = tokenizer.convert_tokens_to_string(tokenizer.convert_ids_to_tokens(inputs[\"input_ids\"][0][answer_start:answer_end]))\n",
        "    return answer\n",
        "# https://huggingface.co/transformers/migration.html\n",
        "def model_evaluation(model, tokenizer, questions, contexts, answers, to_print = False):\n",
        "    preds = []\n",
        "    my_list = list(range(len(questions)))\n",
        "    with tqdm(total=len(my_list)) as pbar:\n",
        "        for ex in range(len(questions)):\n",
        "            answer = get_answer(model, tokenizer, contexts[ex],questions[ex])\n",
        "            preds.append(answer)\n",
        "            if ex % 100 == 0 and to_print:\n",
        "                print(\"context \" +contexts[ex] )\n",
        "                print(\"quest \" +questions[ex] )\n",
        "                print(\"truth \" +answers[ex]['text'] )\n",
        "                print(\"pred \" + answer)\n",
        "            pbar.update(1)\n",
        "    truths = [answers[i]['text'] for i in range(len(answers))]\n",
        "    scores = evaluate_truth_pred(truths, preds)\n",
        "    print(scores)\n",
        "    return scores\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LjLzhebZqo8w",
        "jupyter": {
          "source_hidden": true
        }
      },
      "outputs": [],
      "source": [
        "from sklearn.metrics.pairwise import rbf_kernel\n",
        "def kernel_similarity(x, y):\n",
        "        kernel_dist = rbf_kernel(x.reshape(1, -1),y.reshape(1, -1))\n",
        "        return kernel_dist[0][0]\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GjyGrNhgzOea"
      },
      "outputs": [],
      "source": [
        "# Get AI predictions\n",
        "ai_teaching_preds = []\n",
        "ai_validation_preds = []\n",
        "ai_val_preds = []\n",
        "ai_train_preds = []\n",
        "\n",
        "with open('predictions_ans_val.json', 'r') as handle:\n",
        "    preds_val = json.load(handle)\n",
        "\n",
        "#with open('preds_train_data.p', 'rb') as handle:\n",
        "#    preds_train = pickle.load(handle)\n",
        "\n",
        "for key, value in preds_val.items():\n",
        "    ai_val_preds.append(value)\n",
        "\n",
        "#for key, value in preds_train.items():\n",
        "#    ai_train_preds.append(value)\n",
        "\n",
        "ai_teaching_preds = ai_val_preds[:START_TEST]\n",
        "ai_validation_preds = ai_val_preds[START_TEST:]\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IAzOaRXXzOec"
      },
      "source": [
        "elements of variation:\n",
        "- yes/no questions vs free-form\n",
        "- type of question\n",
        "- lenght of context\n",
        "- LDA topic model, by top model\n",
        "- question classification\n",
        "- embeddings cluster by k-medoids"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X4aLizxtzOec"
      },
      "source": [
        "## Free form vs yes/no"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "APkZ8r2HzOec",
        "outputId": "f809de14-01e9-4f7c-b6a0-41b340bb48e8"
      },
      "outputs": [],
      "source": [
        "# Free form vs yes/no\n",
        "all_indices = set(list(range(len(validation_dataset))))\n",
        "yes_no_indices = set([i  for i in range(len(validation_dataset)) if validation_dataset[i]['answer'] in [\"yes\",\"no\"] ])\n",
        "free_indices = all_indices - yes_no_indices\n",
        "yes_no_truths = [validation_dataset[i]['answer'] for i in yes_no_indices]\n",
        "yes_no_preds = [ai_val_preds[i] for i in yes_no_indices]\n",
        "free_form_truths = [validation_dataset[i]['answer'] for i in all_indices - yes_no_indices]\n",
        "free_form_preds = [ai_val_preds[i] for i in all_indices - yes_no_indices]\n",
        "yes_no_metrics = evaluate_truth_pred(yes_no_truths, yes_no_preds)\n",
        "free_metrics = evaluate_truth_pred(free_form_truths, free_form_preds)\n",
        "print(\"Yes/no questions\")\n",
        "print(yes_no_metrics)\n",
        "print(\"Free form questions\")\n",
        "print(free_metrics)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jBY744DEzOed",
        "jupyter": {
          "source_hidden": true
        }
      },
      "source": [
        "| Variation      | Em | F1 |\n",
        "| ----------- | ----------- | ----------- |\n",
        "| Free form      | 65.81       | 79.86       |\n",
        "| Yes/no   | NA        | NA       |"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n7ejuB3UzOed",
        "jupyter": {
          "source_hidden": true
        }
      },
      "source": [
        "Restrict to only free form questions:"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AILboUv-zOee"
      },
      "source": [
        "## Question type"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "YJ6u6nRVzOee",
        "jupyter": {
          "outputs_hidden": true,
          "source_hidden": true
        },
        "outputId": "41bdfac2-3b76-497b-ab96-f41ac116c942"
      },
      "outputs": [],
      "source": [
        "# question types\n",
        "question_types = set([validation_dataset[i]['type'] for i in free_indices])\n",
        "truths_types = {}\n",
        "preds_types = {}\n",
        "for q_type in question_types:\n",
        "    truths_types[q_type] = []\n",
        "    preds_types[q_type] = []\n",
        "for i in free_indices:\n",
        "    truths_types[validation_dataset[i]['type']].append(validation_dataset[i]['answer'])\n",
        "    preds_types[validation_dataset[i]['type']].append(ai_val_preds[i])\n",
        "\n",
        "for q_type in question_types:\n",
        "    print(\"for type \" +str(q_type) )\n",
        "    metrics = evaluate_truth_pred(truths_types[q_type], preds_types[q_type])\n",
        "    print(metrics)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hUV4EXj-zOee",
        "jupyter": {
          "source_hidden": true
        }
      },
      "source": [
        "| Variation      | Em | F1 |\n",
        "| ----------- | ----------- | ----------- |\n",
        "| bridge      | 64.95       | 79.995       |\n",
        "| comparision  | 70.75        | 79.10       |"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R5v1ArUKzOee"
      },
      "source": [
        "## Passage Lenghts"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "mSF3VYXmzOef",
        "jupyter": {
          "outputs_hidden": true,
          "source_hidden": true
        },
        "outputId": "c0f6071e-9873-40f2-a7af-50aa66dddc10"
      },
      "outputs": [],
      "source": [
        "# passage lenghts\n",
        "scores_em = []\n",
        "scores_f1 = []\n",
        "lenghts = []\n",
        "for i in free_indices:\n",
        "    exact_match = metric_max_over_ground_truths(\n",
        "        exact_match_score, ai_val_preds[i], [validation_dataset[i]['answer']])\n",
        "    f1 = metric_max_over_ground_truths(\n",
        "        f1_score,  ai_val_preds[i], [validation_dataset[i]['answer']])\n",
        "    scores_em.append(exact_match)\n",
        "    scores_f1.append(f1)\n",
        "    lenghts.append(len(validation_dataset[i]['passage']))\n",
        "from scipy import stats\n",
        "n_bins = 10\n",
        "import matplotlib.pyplot as plt\n",
        "y , x, _ = stats.binned_statistic(lenghts, scores_em, 'mean', bins=n_bins)\n",
        "fig, ax = plt.subplots(1)\n",
        "plt.bar(list(range(n_bins)),y)\n",
        "\n",
        "ax.set(ylim=(0.62, 0.72))\n",
        "plt.xticks(list(range(n_bins)), [f'[{x[i]:.0f}, {x[i+1]:.0f}]' for i in range(n_bins)], rotation=\"vertical\")\n",
        "plt.ylabel('EM')\n",
        "plt.xlabel('Passage lenghts')\n",
        "\n",
        "plt.show()\n",
        "fig, ax = plt.subplots(1)\n",
        "y , x, _ = stats.binned_statistic(lenghts, scores_f1, 'mean', bins=n_bins)\n",
        "plt.bar(list(range(n_bins)),y)\n",
        "ax.set(ylim=(0.7, 0.88))\n",
        "\n",
        "plt.xticks(list(range(n_bins)), [f'[{x[i]:.0f}, {x[i+1]:.0f}]' for i in range(n_bins)], rotation=\"vertical\")\n",
        "plt.ylabel('F1')\n",
        "plt.xlabel('Passage lenghts')\n",
        "plt.show()\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eeinT7HozOeg"
      },
      "source": [
        "## Supporting passage lenghts"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "bF8qr4GBzOeg",
        "jupyter": {
          "outputs_hidden": true,
          "source_hidden": true
        },
        "outputId": "d2af11da-6793-4b4f-efa2-ab9baa9a603c"
      },
      "outputs": [],
      "source": [
        "\n",
        "\n",
        "#Our sentences we like to encode\n",
        "lenghts = set([len(validation_dataset[i]['supporting_facts']['sent_id']) for i in free_indices])\n",
        "SUP_FACTS_LEN = len(lenghts)\n",
        "scores_em = [[] for _ in range(SUP_FACTS_LEN)]\n",
        "scores_f1 = [[] for _ in range(SUP_FACTS_LEN)]\n",
        "j = 0\n",
        "for i in free_indices:\n",
        "    exact_match = metric_max_over_ground_truths(\n",
        "        exact_match_score, ai_val_preds[i], [validation_dataset[i]['answer']])\n",
        "    f1 = metric_max_over_ground_truths(\n",
        "        f1_score,  ai_val_preds[i], [validation_dataset[i]['answer']])\n",
        "    cluster_index = len(validation_dataset[i]['supporting_facts']['sent_id']) - min(lenghts)\n",
        "    scores_em[cluster_index].append(exact_match)\n",
        "    scores_f1[cluster_index].append(f1)\n",
        "    j += 1\n",
        "\n",
        "scores_em_avgs = []\n",
        "scores_f1_avgs = []\n",
        "scores_em_stds = []\n",
        "scores_f1_stds = []\n",
        "for i in range(SUP_FACTS_LEN):\n",
        "    scores_em_avgs.append(np.average(scores_em[i]))\n",
        "    scores_f1_avgs.append(np.average(scores_f1[i]))\n",
        "    interval_em = stats.t.interval(alpha=0.95, df=len(scores_em[i])-1, loc=np.mean(scores_em[i]), scale=st.sem(scores_em[i]))\n",
        "    scores_em_stds.append((interval_em[0]-interval_em[1])/2)\n",
        "    interval_f1 = stats.t.interval(alpha=0.95, df=len(scores_f1[i])-1, loc=np.mean(scores_f1[i]), scale=st.sem(scores_f1[i]))\n",
        "    scores_f1_stds.append((interval_f1[1]-interval_f1[0])/2)\n",
        "\n",
        "fig, ax = plt.subplots(1)\n",
        "\n",
        "plt.bar(list(range(SUP_FACTS_LEN)),scores_em_avgs, yerr=scores_em_stds)\n",
        "\n",
        "#ax.set(ylim=(0.2, 0.8))\n",
        "plt.xticks(list(range(SUP_FACTS_LEN)), [f'{i+ min(lenghts)}  {len(scores_f1[i])}' for i in range(SUP_FACTS_LEN)], rotation=\"vertical\")\n",
        "plt.ylabel('EM')\n",
        "plt.xlabel('Lenght of supporting facts')\n",
        "\n",
        "plt.show()\n",
        "fig, ax = plt.subplots(1)\n",
        "plt.bar(list(range(SUP_FACTS_LEN)),scores_f1_avgs, yerr=scores_f1_stds)\n",
        "#ax.set(ylim=(0.4, 0.95))\n",
        "\n",
        "plt.xticks(list(range(SUP_FACTS_LEN)), [f'{i+ min(lenghts)}  {len(scores_f1[i])}' for i in range(SUP_FACTS_LEN)], rotation=\"vertical\")\n",
        "plt.ylabel('F1')\n",
        "plt.xlabel('Lenght of supporting facts')\n",
        "plt.show()\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Mga_3R9VzOeg"
      },
      "source": [
        "## LDA topics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "14hVslHL_0ng",
        "jupyter": {
          "source_hidden": true
        }
      },
      "outputs": [],
      "source": [
        "# Learn LDA model on validation dataset\n",
        "# downloads\n",
        "spacy.load('en')\n",
        "parser = English()\n",
        "nltk.download('averaged_perceptron_tagger')\n",
        "nltk.download('wordnet')\n",
        "nltk.download('stopwords')\n",
        "en_stop = set(nltk.corpus.stopwords.words('english'))\n",
        "nltk.download('punkt')\n",
        "LDA_TOPICS = 10\n",
        "\n",
        "# see https://www.machinelearningplus.com/nlp/topic-modeling-gensim-python/\n",
        "#import pyLDAvis.gensim and https://towardsdatascience.com/topic-modelling-in-python-with-nltk-and-gensim-4ef03213cd21\n",
        "def tokenize(text):\n",
        "    lda_tokens = []\n",
        "    tokens = parser(text)\n",
        "    for token in tokens:\n",
        "        if token.orth_.isspace():\n",
        "            continue\n",
        "        elif token.like_url:\n",
        "            lda_tokens.append('URL')\n",
        "        elif token.orth_.startswith('@'):\n",
        "            lda_tokens.append('SCREEN_NAME')\n",
        "        else:\n",
        "            lda_tokens.append(token.lower_)\n",
        "    return lda_tokens\n",
        "\n",
        "def get_lemma(word):\n",
        "    lemma = wn.morphy(word)\n",
        "    if lemma is None:\n",
        "        return word\n",
        "    else:\n",
        "        return lemma\n",
        "def get_lemma2(word):\n",
        "    return WordNetLemmatizer().lemmatize(word)\n",
        "\n",
        "def detect_question(token):\n",
        "    # assume token is string\n",
        "    tag = nltk.pos_tag([token])\n",
        "    if tag[0][1] in [\"WP\", \"WRB\", \"WP$\"]:\n",
        "        return True\n",
        "    else:\n",
        "        return False\n",
        "def prepare_text_for_lda(text):\n",
        "    tokens = tokenize(text)\n",
        "    tokens = [token for token in tokens if len(token) > 3]\n",
        "    tokens = [token for token in tokens if (token not in en_stop) or  (detect_question(token))]\n",
        "    tokens = [get_lemma(token) for token in tokens]\n",
        "    return tokens\n",
        "\n",
        "def learn_lda_model(dataset):\n",
        "    paragraphs = []\n",
        "    for i in free_indices:\n",
        "        paragraphs.append(dataset[i]['passage'])\n",
        "\n",
        "    # prep data\n",
        "    paragraphs_lda = []\n",
        "    for i in range(len(paragraphs)):\n",
        "        paragraphs_lda.append(prepare_text_for_lda(paragraphs[i]))\n",
        "\n",
        "    dictionary = corpora.Dictionary(paragraphs_lda)\n",
        "    corpus = [dictionary.doc2bow(text) for text in paragraphs_lda]\n",
        "    pickle.dump(corpus, open('corpus.pkl', 'wb'))\n",
        "    dictionary.save('dictionary.gensim')\n",
        "    ldamodel = gensim.models.ldamodel.LdaModel(corpus, num_topics = LDA_TOPICS, id2word=dictionary, passes=15)\n",
        "    ldamodel.save('lda_model_10.gensim')\n",
        "    topics = ldamodel.print_topics(num_words=10)\n",
        "    for topic in topics:\n",
        "        print(topic)\n",
        "\n",
        "class LDA_model:\n",
        "    def __init__(self):\n",
        "        self.n_topics = LDA_TOPICS\n",
        "        self.model =  models.LdaModel.load('lda_model_10.gensim')\n",
        "        self.dictionary = gensim.corpora.Dictionary.load('dictionary.gensim')\n",
        "        self.corpus = pickle.load(open('corpus.pkl', 'rb'))\n",
        "    def predict(self, paragraph):\n",
        "        paragraph = prepare_text_for_lda(paragraph)\n",
        "        new_doc_bow = self.dictionary.doc2bow(paragraph)\n",
        "        result = self.model.get_document_topics(new_doc_bow)\n",
        "        vector = [0] * self.n_topics\n",
        "        for top in result:\n",
        "            vector[top[0]] = top[1]\n",
        "        return vector\n",
        "    def show_topics(self):\n",
        "        topics = self.model.print_topics(num_words=10)\n",
        "        for topic in topics:\n",
        "            print(topic)\n",
        "#learn_lda_model(validation_dataset)\n",
        "lda_model = LDA_model()\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "kCj3k41VzOeh",
        "jupyter": {
          "outputs_hidden": true,
          "source_hidden": true
        },
        "outputId": "e6734e6f-915f-436d-a22b-2ed949077ea8"
      },
      "outputs": [],
      "source": [
        "# LDA differences\n",
        "scores_em = [[] for _ in range(LDA_TOPICS)]\n",
        "scores_f1 = [[] for _ in range(LDA_TOPICS)]\n",
        "for i in free_indices:\n",
        "    exact_match = metric_max_over_ground_truths(\n",
        "        exact_match_score, ai_val_preds[i], [validation_dataset[i]['answer']])\n",
        "    f1 = metric_max_over_ground_truths(\n",
        "        f1_score,  ai_val_preds[i], [validation_dataset[i]['answer']])\n",
        "    topics_passage = lda_model.predict(validation_dataset[i]['passage'])\n",
        "    main_topic = np.argmax(topics_passage)\n",
        "    scores_em[main_topic].append(exact_match)\n",
        "    scores_f1[main_topic].append(f1)\n",
        "\n",
        "scores_em_avgs = []\n",
        "scores_f1_avgs = []\n",
        "for i in range(LDA_TOPICS):\n",
        "    scores_em_avgs.append(np.average(scores_em[i]))\n",
        "    scores_f1_avgs.append(np.average(scores_f1[i]))\n",
        "fig, ax = plt.subplots(1)\n",
        "\n",
        "plt.bar(list(range(LDA_TOPICS)),scores_em_avgs)\n",
        "\n",
        "ax.set(ylim=(0.45, 0.75))\n",
        "#plt.xticks(list(range(LDA_TOPICS)), [f'[{x[i]:.0f}, {x[i+1]:.0f}]' for i in range(LDA_TOPICS)], rotation=\"vertical\")\n",
        "plt.ylabel('EM')\n",
        "plt.xlabel('Topic')\n",
        "\n",
        "plt.show()\n",
        "fig, ax = plt.subplots(1)\n",
        "plt.bar(list(range(LDA_TOPICS)),scores_f1_avgs)\n",
        "ax.set(ylim=(0.65, 0.85))\n",
        "\n",
        "#plt.xticks(list(range(LDA_TOPICS)), [f'[{x[i]:.0f}, {x[i+1]:.0f}]' for i in range(LDA_TOPICS)], rotation=\"vertical\")\n",
        "plt.ylabel('F1')\n",
        "plt.xlabel('Topic')\n",
        "plt.show()\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dnMD3AJVzOeh"
      },
      "source": [
        "## Question Classification"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RVKmWPEyzOei",
        "jupyter": {
          "source_hidden": true
        }
      },
      "outputs": [],
      "source": [
        "\n",
        "def tokenize(text):\n",
        "    lda_tokens = []\n",
        "    tokens = parser(text)\n",
        "    for token in tokens:\n",
        "        if token.orth_.isspace():\n",
        "            continue\n",
        "        elif token.like_url:\n",
        "            lda_tokens.append('URL')\n",
        "        elif token.orth_.startswith('@'):\n",
        "            lda_tokens.append('SCREEN_NAME')\n",
        "        else:\n",
        "            lda_tokens.append(token.lower_)\n",
        "    return lda_tokens\n",
        "\n",
        "def get_lemma(word):\n",
        "    lemma = wn.morphy(word)\n",
        "    if lemma is None:\n",
        "        return word\n",
        "    else:\n",
        "        return lemma\n",
        "def get_lemma2(word):\n",
        "    return WordNetLemmatizer().lemmatize(word)\n",
        "\n",
        "def detect_question(token):\n",
        "    # assume token is string\n",
        "    tag = nltk.pos_tag([token])\n",
        "    if tag[0][1] in [\"WP\", \"WRB\", \"WP$\"]:\n",
        "        return True\n",
        "    else:\n",
        "        return False\n",
        "\n",
        "question_words = [\"what\", \"when\", \"where\", \"which\", \"who\", \"whom\", \"whose\", \"why\", \"how\"]\n",
        "\n",
        "def question_classification(question):\n",
        "    question_vec = [0] * len(question_words)\n",
        "    tokens = tokenize(question)\n",
        "    tokens = [get_lemma(token) for token in tokens]\n",
        "    all_zero = True\n",
        "    for token in tokens:\n",
        "        for i in range(len(question_words)):\n",
        "            if token == question_words[i]:\n",
        "                question_vec[i] = 1\n",
        "                all_zero = False\n",
        "    #if all_zero:\n",
        "    #    print(f' question: {question}')\n",
        "    return question_vec, all_zero\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "IvZu9j3zzOei",
        "jupyter": {
          "outputs_hidden": true,
          "source_hidden": true
        },
        "outputId": "9184a877-9091-4aac-b63a-3aedc27885cd"
      },
      "outputs": [],
      "source": [
        "# question differences\n",
        "QUESTION_WORDS = len(question_words) + 1\n",
        "scores_em = [[] for _ in range(QUESTION_WORDS)]\n",
        "scores_f1 = [[] for _ in range(QUESTION_WORDS)]\n",
        "for i in free_indices:\n",
        "    exact_match = metric_max_over_ground_truths(\n",
        "        exact_match_score, ai_val_preds[i], [validation_dataset[i]['answer']])\n",
        "    f1 = metric_max_over_ground_truths(\n",
        "        f1_score,  ai_val_preds[i], [validation_dataset[i]['answer']])\n",
        "    question_word, all_zero = question_classification(validation_dataset[i]['question'])\n",
        "    if all_zero:\n",
        "        scores_em[-1].append(exact_match)\n",
        "        scores_f1[-1].append(f1)\n",
        "    else:\n",
        "        for j in range(QUESTION_WORDS-1):\n",
        "            if question_word[j] == 1:\n",
        "                scores_em[j].append(exact_match)\n",
        "                scores_f1[j].append(f1)\n",
        "\n",
        "scores_em_avgs = []\n",
        "scores_f1_avgs = []\n",
        "scores_em_stds = []\n",
        "scores_f1_stds = []\n",
        "for i in range(QUESTION_WORDS):\n",
        "    scores_em_avgs.append(np.average(scores_em[i]))\n",
        "    scores_f1_avgs.append(np.average(scores_f1[i]))\n",
        "    scores_em_stds.append(np.std(scores_em[i]))\n",
        "    scores_f1_stds.append(np.std(scores_f1[i]))\n",
        "\n",
        "fig, ax = plt.subplots(1)\n",
        "\n",
        "plt.bar(list(range(QUESTION_WORDS)),scores_em_avgs, yerr=scores_em_stds)\n",
        "\n",
        "ax.set(ylim=(0.2, 0.8))\n",
        "plt.xticks(list(range(QUESTION_WORDS)), [f'{question_words[i]}  {len(scores_f1[i])}' for i in range(QUESTION_WORDS-1)]+[\"other\"], rotation=\"vertical\")\n",
        "plt.ylabel('EM')\n",
        "plt.xlabel('Question word')\n",
        "\n",
        "plt.show()\n",
        "fig, ax = plt.subplots(1)\n",
        "plt.bar(list(range(QUESTION_WORDS)),scores_f1_avgs, yerr=scores_f1_stds)\n",
        "ax.set(ylim=(0.4, 0.95))\n",
        "\n",
        "plt.xticks(list(range(QUESTION_WORDS)), [f'{question_words[i]}  {len(scores_f1[i])}' for i in range(QUESTION_WORDS-1)]+[\"other\"], rotation=\"vertical\")\n",
        "plt.ylabel('F1')\n",
        "plt.xlabel('Question word')\n",
        "plt.show()\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KWmZHFwqzOei"
      },
      "source": [
        "## Model embeddings clustering\n",
        "- average embeddings from model\n",
        "- embeddings from questions, sentence embeddings\n",
        "- embeddings from passages\n",
        "- embeddings from answers\n",
        "compare both k-means and k-medoids\n",
        "```{toggle}\n",
        "\n",
        "```\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6NWUJ_RvzOei",
        "jupyter": {
          "source_hidden": true
        }
      },
      "source": [
        "### Average from QA model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9AU-3qlTzOej",
        "jupyter": {
          "outputs_hidden": true,
          "source_hidden": true
        },
        "outputId": "86bab911-dc6f-479d-fd24-bfa983694dab"
      },
      "outputs": [],
      "source": [
        "# question differences\n",
        "from sklearn.cluster import KMeans\n",
        "import pickle\n",
        "with open('C:/Users/Hussein/Documents/Research/Human learn to defer/hotpotqa/dev set preds/embeddings_data_val.p', 'rb') as handle:\n",
        "    embeddings_val = pickle.load(handle)\n",
        "\n",
        "embeddings_val = [embeddings_val[i][1] for i in range(len(embeddings_val))]\n",
        "embeddings_val = np.asarray(embeddings_val)\n",
        "teaching_embeddings = embeddings_val[:START_TEST]\n",
        "validation_embeddings = embeddings_val[START_TEST:]\n",
        "\n",
        "\n",
        "N_CLUSTERS = 13\n",
        "kmeans = KMeans(n_clusters=N_CLUSTERS).fit(embeddings_val)\n",
        "print(f'inertia is {kmeans.inertia_:.3f}')\n",
        "scores_em = [[] for _ in range(N_CLUSTERS)]\n",
        "scores_f1 = [[] for _ in range(N_CLUSTERS)]\n",
        "for i in free_indices:\n",
        "    exact_match = metric_max_over_ground_truths(\n",
        "        exact_match_score, ai_val_preds[i], [validation_dataset[i]['answer']])\n",
        "    f1 = metric_max_over_ground_truths(\n",
        "        f1_score,  ai_val_preds[i], [validation_dataset[i]['answer']])\n",
        "    cluster_index = kmeans.labels_[i]\n",
        "    scores_em[cluster_index].append(exact_match)\n",
        "    scores_f1[cluster_index].append(f1)\n",
        "\n",
        "scores_em_avgs = []\n",
        "scores_f1_avgs = []\n",
        "scores_em_stds = []\n",
        "scores_f1_stds = []\n",
        "for i in range(N_CLUSTERS):\n",
        "    scores_em_avgs.append(np.average(scores_em[i]))\n",
        "    scores_f1_avgs.append(np.average(scores_f1[i]))\n",
        "    interval_em = stats.t.interval(alpha=0.95, df=len(scores_em[i])-1, loc=np.mean(scores_em[i]), scale=st.sem(scores_em[i]))\n",
        "    scores_em_stds.append((interval_em[0]-interval_em[1])/2)\n",
        "    interval_f1 = stats.t.interval(alpha=0.95, df=len(scores_f1[i])-1, loc=np.mean(scores_f1[i]), scale=st.sem(scores_f1[i]))\n",
        "    scores_f1_stds.append((interval_em[1]-interval_em[1])/2)\n",
        "\n",
        "fig, ax = plt.subplots(1)\n",
        "\n",
        "plt.bar(list(range(N_CLUSTERS)),scores_em_avgs, yerr=scores_em_stds)\n",
        "\n",
        "#ax.set(ylim=(0.2, 0.8))\n",
        "plt.xticks(list(range(N_CLUSTERS)), [f'{i}  {len(scores_f1[i])}' for i in range(N_CLUSTERS)], rotation=\"vertical\")\n",
        "plt.ylabel('EM')\n",
        "plt.xlabel('Cluster index')\n",
        "\n",
        "plt.show()\n",
        "fig, ax = plt.subplots(1)\n",
        "plt.bar(list(range(N_CLUSTERS)),scores_f1_avgs, yerr=scores_f1_stds)\n",
        "#ax.set(ylim=(0.4, 0.95))\n",
        "\n",
        "plt.xticks(list(range(N_CLUSTERS)), [f'{i}  {len(scores_f1[i])}' for i in range(N_CLUSTERS)], rotation=\"vertical\")\n",
        "plt.ylabel('F1')\n",
        "plt.xlabel('Cluster index')\n",
        "plt.show()\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8KD-kLfezOej"
      },
      "source": [
        "### Embedding Passage"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NL8HREvpzOej",
        "jupyter": {
          "source_hidden": true
        }
      },
      "outputs": [],
      "source": [
        "from sentence_transformers import SentenceTransformer\n",
        "model = SentenceTransformer('distilbert-base-nli-stsb-mean-tokens')\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "1E3myLyIzOej",
        "jupyter": {
          "outputs_hidden": true,
          "source_hidden": true
        },
        "outputId": "6d914ae7-5371-4ae0-d123-1d5e33b815db"
      },
      "outputs": [],
      "source": [
        "\n",
        "\n",
        "#Our sentences we like to encode\n",
        "#Our sentences we like to encode\n",
        "all_passages = [validation_dataset[i]['passage'] for i in free_indices]\n",
        "\n",
        "#Sentences are encoded by calling model.encode()\n",
        "embeddings_passage = model.encode(all_passages)\n",
        "\n",
        "N_CLUSTERS = 10\n",
        "kmeans = KMeans(n_clusters=N_CLUSTERS,max_iter = 10000).fit(embeddings_passage)\n",
        "print(f'inertia is {kmeans.inertia_:.3f}')\n",
        "scores_em = [[] for _ in range(N_CLUSTERS)]\n",
        "scores_f1 = [[] for _ in range(N_CLUSTERS)]\n",
        "j = 0\n",
        "for i in free_indices:\n",
        "    exact_match = metric_max_over_ground_truths(\n",
        "        exact_match_score, ai_val_preds[i], [validation_dataset[i]['answer']])\n",
        "    f1 = metric_max_over_ground_truths(\n",
        "        f1_score,  ai_val_preds[i], [validation_dataset[i]['answer']])\n",
        "    cluster_index = kmeans.labels_[j]\n",
        "    scores_em[cluster_index].append(exact_match)\n",
        "    scores_f1[cluster_index].append(f1)\n",
        "    j += 1\n",
        "\n",
        "scores_em_avgs = []\n",
        "scores_f1_avgs = []\n",
        "scores_em_stds = []\n",
        "scores_f1_stds = []\n",
        "for i in range(N_CLUSTERS):\n",
        "    scores_em_avgs.append(np.average(scores_em[i]))\n",
        "    scores_f1_avgs.append(np.average(scores_f1[i]))\n",
        "    interval_em = stats.t.interval(alpha=0.95, df=len(scores_em[i])-1, loc=np.mean(scores_em[i]), scale=st.sem(scores_em[i]))\n",
        "    scores_em_stds.append((interval_em[0]-interval_em[1])/2)\n",
        "    interval_f1 = stats.t.interval(alpha=0.95, df=len(scores_f1[i])-1, loc=np.mean(scores_f1[i]), scale=st.sem(scores_f1[i]))\n",
        "    scores_f1_stds.append((interval_f1[1]-interval_f1[0])/2)\n",
        "\n",
        "fig, ax = plt.subplots(1)\n",
        "\n",
        "plt.bar(list(range(N_CLUSTERS)),scores_em_avgs, yerr=scores_em_stds)\n",
        "\n",
        "#ax.set(ylim=(0.2, 0.8))\n",
        "plt.xticks(list(range(N_CLUSTERS)), [f'{i}  {len(scores_f1[i])}' for i in range(N_CLUSTERS)], rotation=\"vertical\")\n",
        "plt.ylabel('EM')\n",
        "plt.xlabel('Cluster index')\n",
        "\n",
        "plt.show()\n",
        "fig, ax = plt.subplots(1)\n",
        "plt.bar(list(range(N_CLUSTERS)),scores_f1_avgs, yerr=scores_f1_stds)\n",
        "#ax.set(ylim=(0.4, 0.95))\n",
        "\n",
        "plt.xticks(list(range(N_CLUSTERS)), [f'{i}  {len(scores_f1[i])}' for i in range(N_CLUSTERS)], rotation=\"vertical\")\n",
        "plt.ylabel('F1')\n",
        "plt.xlabel('Cluster index')\n",
        "plt.show()\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bhmcbcPwzOek"
      },
      "source": [
        "### Embedding Question"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "f6F3K8NSzOek",
        "jupyter": {
          "outputs_hidden": true,
          "source_hidden": true
        },
        "outputId": "7c372f71-7bd3-43d8-de61-61420730cdc3"
      },
      "outputs": [],
      "source": [
        "\n",
        "#Our sentences we like to encode\n",
        "all_passages = [validation_dataset[i]['question'] for i in free_indices]\n",
        "\n",
        "#Sentences are encoded by calling model.encode()\n",
        "embeddings_passage = model.encode(all_passages)\n",
        "\n",
        "N_CLUSTERS = 40\n",
        "kmeans = KMeans(n_clusters=N_CLUSTERS,max_iter = 10000).fit(embeddings_passage)\n",
        "print(f'inertia is {kmeans.inertia_:.3f}')\n",
        "scores_em = [[] for _ in range(N_CLUSTERS)]\n",
        "scores_f1 = [[] for _ in range(N_CLUSTERS)]\n",
        "j = 0\n",
        "for i in free_indices:\n",
        "    exact_match = metric_max_over_ground_truths(\n",
        "        exact_match_score, ai_val_preds[i], [validation_dataset[i]['answer']])\n",
        "    f1 = metric_max_over_ground_truths(\n",
        "        f1_score,  ai_val_preds[i], [validation_dataset[i]['answer']])\n",
        "    cluster_index = kmeans.labels_[j]\n",
        "    scores_em[cluster_index].append(exact_match)\n",
        "    scores_f1[cluster_index].append(f1)\n",
        "    j += 1\n",
        "scores_em_avgs = []\n",
        "scores_f1_avgs = []\n",
        "scores_em_stds = []\n",
        "scores_f1_stds = []\n",
        "for i in range(N_CLUSTERS):\n",
        "    scores_em_avgs.append(np.average(scores_em[i]))\n",
        "    scores_f1_avgs.append(np.average(scores_f1[i]))\n",
        "    interval_em = stats.t.interval(alpha=0.95, df=len(scores_em[i])-1, loc=np.mean(scores_em[i]), scale=st.sem(scores_em[i]))\n",
        "    scores_em_stds.append((interval_em[0]-interval_em[1])/2)\n",
        "    interval_f1 = stats.t.interval(alpha=0.95, df=len(scores_f1[i])-1, loc=np.mean(scores_f1[i]), scale=st.sem(scores_f1[i]))\n",
        "    scores_f1_stds.append((interval_f1[1]-interval_f1[0])/2)\n",
        "\n",
        "fig, ax = plt.subplots(1)\n",
        "\n",
        "plt.bar(list(range(N_CLUSTERS)),scores_em_avgs, yerr=scores_em_stds)\n",
        "\n",
        "#ax.set(ylim=(0.2, 0.8))\n",
        "plt.xticks(list(range(N_CLUSTERS)), [f'{i}  {len(scores_f1[i])}' for i in range(N_CLUSTERS)], rotation=\"vertical\")\n",
        "plt.ylabel('EM')\n",
        "plt.xlabel('Cluster index')\n",
        "\n",
        "plt.show()\n",
        "fig, ax = plt.subplots(1)\n",
        "plt.bar(list(range(N_CLUSTERS)),scores_f1_avgs, yerr=scores_f1_stds)\n",
        "#ax.set(ylim=(0.4, 0.95))\n",
        "\n",
        "plt.xticks(list(range(N_CLUSTERS)), [f'{i}  {len(scores_f1[i])}' for i in range(N_CLUSTERS)], rotation=\"vertical\")\n",
        "plt.ylabel('F1')\n",
        "plt.xlabel('Cluster index')\n",
        "plt.show()\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "Bi3mXNTkzOek",
        "jupyter": {
          "outputs_hidden": true,
          "source_hidden": true
        },
        "outputId": "84491b73-ec24-4830-e56b-7834637e1eb8"
      },
      "outputs": [],
      "source": [
        "plt.figure(figsize=(40,10))\n",
        "plt.bar(list(range(N_CLUSTERS)),scores_f1_avgs, yerr=scores_f1_stds)\n",
        "#ax.set(ylim=(0.4, 0.95))\n",
        "\n",
        "plt.xticks(list(range(N_CLUSTERS)), [f'{i}  {len(scores_f1[i])}' for i in range(N_CLUSTERS)], rotation=\"vertical\")\n",
        "plt.ylabel('F1')\n",
        "plt.xlabel('Cluster index')\n",
        "plt.show()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SCcjKJJZzOel",
        "jupyter": {
          "source_hidden": true
        }
      },
      "outputs": [],
      "source": [
        "j = 0\n",
        "for i in free_indices:\n",
        "\n",
        "    cluster_index = kmeans.labels_[j]\n",
        "    if cluster_index == 12:\n",
        "        print(validation_dataset[i]['question'])\n",
        "    j += 1"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZwWEWba9zOel"
      },
      "source": [
        "### Embedding Answer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "Mhi5D2aCzOel",
        "jupyter": {
          "outputs_hidden": true,
          "source_hidden": true
        },
        "outputId": "7a00f594-972f-49dc-acf3-f2c2229f8875"
      },
      "outputs": [],
      "source": [
        "\n",
        "#Our sentences we like to encode\n",
        "all_passages = [validation_dataset[i]['answer'] for i in free_indices]\n",
        "\n",
        "#Sentences are encoded by calling model.encode()\n",
        "embeddings_passage = model.encode(all_passages)\n",
        "\n",
        "N_CLUSTERS = 40\n",
        "kmeans = KMeans(n_clusters=N_CLUSTERS,max_iter = 10000).fit(embeddings_passage)\n",
        "print(f'inertia is {kmeans.inertia_:.3f}')\n",
        "scores_em = [[] for _ in range(N_CLUSTERS)]\n",
        "scores_f1 = [[] for _ in range(N_CLUSTERS)]\n",
        "j = 0\n",
        "for i in free_indices:\n",
        "    exact_match = metric_max_over_ground_truths(\n",
        "        exact_match_score, ai_val_preds[i], [validation_dataset[i]['answer']])\n",
        "    f1 = metric_max_over_ground_truths(\n",
        "        f1_score,  ai_val_preds[i], [validation_dataset[i]['answer']])\n",
        "    cluster_index = kmeans.labels_[j]\n",
        "    scores_em[cluster_index].append(exact_match)\n",
        "    scores_f1[cluster_index].append(f1)\n",
        "    j += 1\n",
        "scores_em_avgs = []\n",
        "scores_f1_avgs = []\n",
        "scores_em_stds = []\n",
        "scores_f1_stds = []\n",
        "for i in range(N_CLUSTERS):\n",
        "    scores_em_avgs.append(np.average(scores_em[i]))\n",
        "    scores_f1_avgs.append(np.average(scores_f1[i]))\n",
        "    interval_em = stats.t.interval(alpha=0.95, df=len(scores_em[i])-1, loc=np.mean(scores_em[i]), scale=st.sem(scores_em[i]))\n",
        "    scores_em_stds.append((interval_em[1]-interval_em[0])/2)\n",
        "    interval_f1 = stats.t.interval(alpha=0.95, df=len(scores_f1[i])-1, loc=np.mean(scores_f1[i]), scale=st.sem(scores_f1[i]))\n",
        "    scores_f1_stds.append((interval_f1[1]-interval_f1[0])/2)\n",
        "\n",
        "fig, ax = plt.subplots(1)\n",
        "\n",
        "plt.bar(list(range(N_CLUSTERS)),scores_em_avgs, yerr=scores_em_stds)\n",
        "\n",
        "ax.set(ylim=(0.4, 0.8))\n",
        "plt.xticks(list(range(N_CLUSTERS)), [f'{i}  {len(scores_f1[i])}' for i in range(N_CLUSTERS)], rotation=\"vertical\")\n",
        "plt.ylabel('EM')\n",
        "plt.xlabel('Cluster index')\n",
        "\n",
        "plt.show()\n",
        "fig, ax = plt.subplots(1)\n",
        "plt.bar(list(range(N_CLUSTERS)),scores_f1_avgs, yerr=scores_f1_stds)\n",
        "#ax.set(ylim=(0.4, 0.95))\n",
        "\n",
        "plt.xticks(list(range(N_CLUSTERS)), [f'{i}  {len(scores_f1[i])}' for i in range(N_CLUSTERS)], rotation=\"vertical\")\n",
        "plt.ylabel('F1')\n",
        "plt.xlabel('Cluster index')\n",
        "plt.show()\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "5OVcDTrnzOel",
        "jupyter": {
          "outputs_hidden": true,
          "source_hidden": true
        },
        "outputId": "d8fc46a0-be09-440e-997e-d0985588b2b2"
      },
      "outputs": [],
      "source": [
        "plt.figure(figsize=(40,10))\n",
        "plt.bar(list(range(N_CLUSTERS)),scores_em_avgs, yerr=scores_em_stds)\n",
        "\n",
        "ax.set(ylim=(0.4, 0.8))\n",
        "plt.xticks(list(range(N_CLUSTERS)), [f'{i}  {len(scores_f1[i])}' for i in range(N_CLUSTERS)], rotation=\"vertical\")\n",
        "plt.ylabel('EM')\n",
        "plt.xlabel('Cluster index')\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8LbKNbJLzOem"
      },
      "source": [
        "# Classifier to predict mistakes\n",
        "Build classifier based on concatenated embeddings of passage, question and AI prediction, predicts If EM=1 or not."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "X--bKAftzOem"
      },
      "outputs": [],
      "source": [
        "#Our sentences we like to encode\n",
        "all_passages = [validation_dataset[i]['passage'] for i in free_indices]\n",
        "all_questions = [validation_dataset[i]['question'] for i in free_indices]\n",
        "all_aipreds = [ai_val_preds[i] for i in free_indices]\n",
        "#Sentences are encoded by calling model.encode()\n",
        "embeddings_passage = model.encode(all_passages)\n",
        "embeddings_question = model.encode(all_questions)\n",
        "embeddings_preds = model.encode(all_aipreds)\n",
        "embeddings = np.asarray([np.concatenate([embeddings_passage[i],embeddings_question[i],embeddings_preds[i]]) for i in range(len(embeddings_passage))])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ySVqghrqzOem"
      },
      "outputs": [],
      "source": [
        "# Optimal deferall decisions\n",
        "labels_ai = []\n",
        "all_answers = [validation_dataset[i]['answer'] for i in free_indices]\n",
        "\n",
        "for i in range(len(all_answers)):\n",
        "    f1_ai = metric_max_over_ground_truths(f1_score,all_answers[i], [all_aipreds[i]])\n",
        "    if f1_ai <0.5:\n",
        "        labels_ai.append(1)\n",
        "    else:\n",
        "        labels_ai.append(0)\n",
        "labels_ai = np.asarray(labels_ai)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-AzN5wdAzOem"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "from sklearn.decomposition import PCA\n",
        "pca = PCA(n_components=200)\n",
        "X_pca = pca.fit_transform(embeddings)\n",
        "print(pca.explained_variance_ratio_)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "1TlEC5TBzOen",
        "jupyter": {
          "outputs_hidden": true,
          "source_hidden": true
        },
        "outputId": "e4b19b82-bd7a-4c0f-c201-7f1c7c35ef79"
      },
      "outputs": [],
      "source": [
        "from matplotlib.colors import ListedColormap\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.datasets import make_moons, make_circles, make_classification\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.gaussian_process import GaussianProcessClassifier\n",
        "from sklearn.gaussian_process.kernels import RBF\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis\n",
        "\n",
        "\n",
        "names = [\"Linear SVM\", \"RBF SVM\",\n",
        "         \"Decision Tree\", \"Random Forest\", \"Neural Net\", \"AdaBoost\",\n",
        "    \"QDA\"]\n",
        "\n",
        "classifiers = [\n",
        "    SVC(kernel=\"linear\", C=0.025),\n",
        "    SVC(gamma=2, C=1),\n",
        "    DecisionTreeClassifier(max_depth=10),\n",
        "    RandomForestClassifier(),\n",
        "    MLPClassifier(alpha=1, max_iter=1000),\n",
        "    AdaBoostClassifier(),\n",
        "    QuadraticDiscriminantAnalysis()]\n",
        "\n",
        "\n",
        "# preprocess dataset, split into training and test part\n",
        "X = X_pca\n",
        "y = labels_ai\n",
        "#X = StandardScaler().fit_transform(X)\n",
        "X_train, X_test, y_train, y_test = \\\n",
        "    train_test_split(X, y, test_size=.3, random_state=42)\n",
        "\n",
        "print(\"Starting classifcation\")\n",
        "# iterate over classifiers\n",
        "for name, clf in zip(names, classifiers):\n",
        "    clf.fit(X_train, y_train)\n",
        "    score = clf.score(X_test, y_test)\n",
        "    print(f'score for {name} is {score:.3f}')\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "p8wpmdMSzOen",
        "jupyter": {
          "source_hidden": true
        }
      },
      "outputs": [],
      "source": [
        "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"distilbert-base-uncased-finetuned-sst-2-english\")\n",
        "\n",
        "model_class = AutoModelForSequenceClassification.from_pretrained(\"distilbert-base-uncased-finetuned-sst-2-english\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-BoSNYLQzOen"
      },
      "outputs": [],
      "source": [
        "all_passages = []\n",
        "all_questions =\n",
        "all_aipreds = []\n",
        "labels = []\n",
        "inputs = []"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pEWcl_tuzOen"
      },
      "outputs": [],
      "source": [
        "train_encodings = tokenizer(train_questions, train_contexts,  truncation=True, padding=True, return_tensors=\"pt\")\n",
        "trainval_encodings = tokenizer(trainval_questions, trainval_contexts,  truncation=True, padding=True, return_tensors=\"pt\")\n",
        "val_encodings = tokenizer(val_questions, val_contexts, truncation=True, padding=True)\n",
        "val_ans_encodings = tokenizer([val_answers[i]['text'] for i in range(len(val_answers))])\n",
        "train_ans_encodings = tokenizer([train_answers[i]['text'] for i in range(len(train_answers))])\n",
        "trainval_ans_encodings = tokenizer([trainval_answers[i]['text'] for i in range(len(trainval_answers))])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7fHmSMdbzOen"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "train_encodings = tokenizer(train_questions, train_contexts,  truncation=True, padding=True, return_tensors=\"pt\")\n",
        "trainval_encodings = tokenizer(trainval_questions, trainval_contexts,  truncation=True, padding=True, return_tensors=\"pt\")\n",
        "val_encodings = tokenizer(val_questions, val_contexts, truncation=True, padding=True)\n",
        "val_ans_encodings = tokenizer([val_answers[i]['text'] for i in range(len(val_answers))])\n",
        "train_ans_encodings = tokenizer([train_answers[i]['text'] for i in range(len(train_answers))])\n",
        "trainval_ans_encodings = tokenizer([trainval_answers[i]['text'] for i in range(len(trainval_answers))])\n",
        "class HotpotDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, encodings):\n",
        "        self.encodings = encodings\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        return {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.encodings.input_ids)\n",
        "BATCH_SIZE = 8\n",
        "train_dataset = HotpotDataset(train_encodings)\n",
        "trainval_dataset = HotpotDataset(trainval_encodings)\n",
        "val_dataset = HotpotDataset(val_encodings)\n",
        "train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=False)\n",
        "trainval_loader = DataLoader(trainval_dataset, batch_size=BATCH_SIZE, shuffle=False)\n",
        "val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gI8JiJdFzOeo",
        "jupyter": {
          "source_hidden": true
        }
      },
      "outputs": [],
      "source": [
        "from transformers import get_linear_schedule_with_warmup\n",
        "n_epochs = 4\n",
        "model.train()\n",
        "optimizer = AdamW(model.parameters(), lr=3e-5)\n",
        "scheduler = get_linear_schedule_with_warmup(optimizer, 1000, len(train_questions)*n_epochs/BATCH_SIZE)\n",
        "\n",
        "from tqdm import tqdm\n",
        "my_list = list(range(len(train_questions)*n_epochs))\n",
        "with tqdm(total=len(my_list)) as pbar:\n",
        "    for epoch in range(n_epochs):\n",
        "        print(\"####################\")\n",
        "        print(\"New epoch\")\n",
        "        print(\"####################\")\n",
        "        model.eval()\n",
        "        model_evaluation(model, tokenizer, trainval_questions, trainval_contexts, trainval_answers)\n",
        "        model.train()\n",
        "        for batch in train_loader:\n",
        "            optimizer.zero_grad()\n",
        "            input_ids = batch['input_ids'].to(device)\n",
        "            attention_mask = batch['attention_mask'].to(device)\n",
        "            start_positions = batch['start_positions'].to(device)\n",
        "            end_positions = batch['end_positions'].to(device)\n",
        "            outputs = model(input_ids, attention_mask=attention_mask, start_positions=start_positions, end_positions=end_positions)\n",
        "            answer_start_scores = outputs['start_logits'][0]\n",
        "            answer_end_scores = outputs['end_logits'][0]\n",
        "            answer_start = torch.argmax(answer_start_scores)  # get the most likely beginning of answer with the argmax of the score\n",
        "            answer_end = torch.argmax(answer_end_scores) + 1  # get the most likely end of answer with the argmax of the score\n",
        "            loss = outputs[0]\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "            scheduler.step()\n",
        "            pbar.update(BATCH_SIZE)\n",
        "model.eval()\n",
        "model_evaluation(model, tokenizer, trainval_questions, trainval_contexts, trainval_answers)\n",
        "#model.eval()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XSH5aTNqzOeo"
      },
      "source": [
        "# Bag of words classifier"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Zxuco-ISzOeo",
        "jupyter": {
          "source_hidden": true
        }
      },
      "outputs": [],
      "source": [
        "y = labels_ai\n",
        "X = [ all_questions[i]  for i in range(len(all_passages))]\n",
        "#X = [all_passages[i] +\" \"+ all_questions[i] +\" \"+ all_aipreds[i] for i in range(len(all_passages))]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sMVS7SvIzOeo",
        "jupyter": {
          "source_hidden": true
        }
      },
      "outputs": [],
      "source": [
        "documents = []\n",
        "\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "\n",
        "stemmer = WordNetLemmatizer()\n",
        "\n",
        "for sen in range(0, len(X)):\n",
        "    # Remove all the special characters\n",
        "    document = re.sub(r'\\W', ' ', str(X[sen]))\n",
        "\n",
        "    # remove all single characters\n",
        "    #document = re.sub(r'\\s+[a-zA-Z]\\s+', ' ', document)\n",
        "\n",
        "    # Remove single characters from the start\n",
        "    document = re.sub(r'\\^[a-zA-Z]\\s+', ' ', document)\n",
        "\n",
        "    # Substituting multiple spaces with single space\n",
        "    document = re.sub(r'\\s+', ' ', document, flags=re.I)\n",
        "\n",
        "    # Removing prefixed 'b'\n",
        "    document = re.sub(r'^b\\s+', '', document)\n",
        "\n",
        "    # Converting to Lowercase\n",
        "    document = document.lower()\n",
        "\n",
        "    # Lemmatization\n",
        "    document = document.split()\n",
        "\n",
        "    document = [stemmer.lemmatize(word) for word in document]\n",
        "    document = ' '.join(document)\n",
        "\n",
        "    documents.append(document)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "qUI-8F7BzOeo",
        "jupyter": {
          "outputs_hidden": true,
          "source_hidden": true
        },
        "outputId": "ac01e9c9-59e1-45ee-8a1a-1f683a6d39b4"
      },
      "outputs": [],
      "source": [
        "documents[0:4]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qICMtHkmzOep",
        "jupyter": {
          "source_hidden": true
        }
      },
      "outputs": [],
      "source": [
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from nltk.corpus import stopwords\n",
        "\n",
        "vectorizer = CountVectorizer(max_features=1500)\n",
        "X = vectorizer.fit_transform(documents).toarray()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4HcVT0tVzOep",
        "jupyter": {
          "source_hidden": true
        }
      },
      "outputs": [],
      "source": [
        "from sklearn.feature_extraction.text import TfidfTransformer\n",
        "tfidfconverter = TfidfTransformer()\n",
        "X = tfidfconverter.fit_transform(X).toarray()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "f9AWYaShzOep",
        "jupyter": {
          "outputs_hidden": true,
          "source_hidden": true
        },
        "outputId": "264159b4-f087-44df-fea4-56eddb19aee1"
      },
      "outputs": [],
      "source": [
        "'''\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "tfidfconverter = TfidfVectorizer(max_features=1500, min_df=5, max_df=0.7, stop_words=stopwords.words('english'))\n",
        "X = tfidfconverter.fit_transform(documents).toarray()\n",
        "'''"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "APCdDnPdzOeq",
        "jupyter": {
          "source_hidden": true
        }
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "9zXyxIA_zOet",
        "jupyter": {
          "outputs_hidden": true,
          "source_hidden": true
        },
        "outputId": "73801f0a-f1bc-426e-de0c-66eeda8f245f"
      },
      "outputs": [],
      "source": [
        "from sklearn.linear_model import LogisticRegression\n",
        "classifier = SVC()\n",
        "classifier.fit(X_train, y_train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bCLvrNGfzOet",
        "jupyter": {
          "source_hidden": true
        }
      },
      "outputs": [],
      "source": [
        "y_pred = classifier.predict(X_test)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "id": "p0gT5jTVzOeu",
        "jupyter": {
          "outputs_hidden": true,
          "source_hidden": true
        },
        "outputId": "cfe4003a-0a05-4405-d76e-c3ceacfd72de"
      },
      "outputs": [],
      "source": [
        "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score\n",
        "\n",
        "print(confusion_matrix(y_test,y_pred))\n",
        "print(classification_report(y_test,y_pred))\n",
        "print(accuracy_score(y_test, y_pred))"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "UP1CACu3irWL",
        "5WSnSiY1dF2E",
        "PuxiBHbj_zde"
      ],
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.7"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "16427970de0f4bf1993057a03ca794f8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "16d94096022c4dea98a298ef76ecc634": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "100%",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c5f078d33bc0446eb28a8e9115e5f4fe",
            "max": 90447,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_558617f2b4674f08bc1eccb0943e5bde",
            "value": 90447
          }
        },
        "437c83dfbf4a48b4a6ac54763cbef34e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "100%",
            "description_tooltip": null,
            "layout": "IPY_MODEL_93753133e53740fab9d14afc1fafc15e",
            "max": 7405,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f99a63e054634f4cb62a9fb6a42d2cb7",
            "value": 7405
          }
        },
        "558617f2b4674f08bc1eccb0943e5bde": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": "initial"
          }
        },
        "5e623b98871a4d9298d7c51908f7214d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_16d94096022c4dea98a298ef76ecc634",
              "IPY_MODEL_c906e0e8c43d4a09b948eef339ca0a09"
            ],
            "layout": "IPY_MODEL_16427970de0f4bf1993057a03ca794f8"
          }
        },
        "60ebf29513bc4fad96ff1dbc5a8f289b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f7c255b791304f7cb95cb8aba66e179c",
            "placeholder": "​",
            "style": "IPY_MODEL_d8e89b135b16432599ff73b8c9694bcb",
            "value": " 7405/7405 [01:32&lt;00:00, 80.45ex/s]"
          }
        },
        "92aa6afcfe1247398e49888b26a6ecfb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_437c83dfbf4a48b4a6ac54763cbef34e",
              "IPY_MODEL_60ebf29513bc4fad96ff1dbc5a8f289b"
            ],
            "layout": "IPY_MODEL_a2f97d589036400494cc7d8c4fa4727d"
          }
        },
        "93753133e53740fab9d14afc1fafc15e": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9c3d199fa1f64dd4a21c3d7bceed52f3": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9d988b67ff1343989cab780a1f3e6c87": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a2f97d589036400494cc7d8c4fa4727d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c5f078d33bc0446eb28a8e9115e5f4fe": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c906e0e8c43d4a09b948eef339ca0a09": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9c3d199fa1f64dd4a21c3d7bceed52f3",
            "placeholder": "​",
            "style": "IPY_MODEL_9d988b67ff1343989cab780a1f3e6c87",
            "value": " 90447/90447 [01:51&lt;00:00, 810.08ex/s]"
          }
        },
        "d8e89b135b16432599ff73b8c9694bcb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f7c255b791304f7cb95cb8aba66e179c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f99a63e054634f4cb62a9fb6a42d2cb7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": "initial"
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}