notes/nlp/nlp_Named-Entity-Recognition-RNN.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "import random\n",
    "from tqdm import tqdm\n",
    "from datetime import datetime\n",
    "from heapq import heappush,heappop\n",
    "from collections import Counter,defaultdict\n",
    "import multiprocessing as mp\n",
    "import tensorflow as tf\n",
    "from sklearn.utils import shuffle\n",
    "import time\n",
    "from tensorflow.keras import Input,Model\n",
    "from tensorflow.keras.layers import Embedding,Bidirectional,LSTM,Dense\n",
    "from tensorflow.keras.losses import CategoricalCrossentropy "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATHglove=\"/home/yui/Documents/data/nlp/glove.6B/glove.6B.50d.txt\"\n",
    "PATHner=\"/home/yui/Documents/data/nlp/ner/ner.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class preprocess:\n",
    "    def __init__(self,PATHglove,PATHner):\n",
    "        self.PATHglove = PATHglove\n",
    "        self.PATHner = PATHner\n",
    "        self.initEmbeddings()\n",
    "        self.V_ = len(self.w2id)\n",
    "        self.k = len(self.res[0])\n",
    "        self.getData()\n",
    "        Lengths = list(map(len,self.sents))\n",
    "        self.maxLength = max(Lengths)+1\n",
    "        self.V = len(self.w2id)\n",
    "        self.sents=[sent+[self.w2id[\"<e>\"]]*(self.maxLength-len(sent)) \n",
    "                    for sent in self.sents]\n",
    "        self.sents=[np.array(sent) for sent in self.sents]\n",
    "        self.labels=[label+[0]*(self.maxLength-len(label)) \n",
    "                     for label in self.labels]\n",
    "        self.labels=[np.array(label) for label in self.labels]\n",
    "    def initEmbeddings(self):\n",
    "        self.w2id,self.id2w,self.res={},{},[]\n",
    "        with open(self.PATHglove) as f:\n",
    "            lines = f.readlines()\n",
    "            for i,line in enumerate(lines):\n",
    "                line = line.strip()\n",
    "                tokens = line.split(\" \")\n",
    "                word = tokens[0]\n",
    "                vec = list(map(float,tokens[1:]))\n",
    "                self.res.append(vec)\n",
    "                self.w2id[word]=i\n",
    "                self.id2w[i]=word\n",
    "        print(\"[INFO] Loading Embeddings ... Done\")\n",
    "        print(\"Vocabulary size: \",len(self.w2id))\n",
    "        print(\"Embeddings: \",self.res[0])\n",
    "        print(\"----------\")\n",
    "    def getData(self):\n",
    "        self.sents,self.labels,self.l2id,self.id2l = [],[],{},{}\n",
    "        j = 0\n",
    "        with open(self.PATHner) as f:\n",
    "            lines = f.readlines()\n",
    "            sent,slab = [],[]\n",
    "            for i,line in enumerate(lines):\n",
    "                line = line.strip()\n",
    "                tokens = line.split(\"\\t\")\n",
    "                word = tokens[0].lower()\n",
    "                if word==\"\":\n",
    "                    self.sents.append(sent)\n",
    "                    self.labels.append(slab)\n",
    "                    sent,slab = [],[]\n",
    "                    continue\n",
    "                label = tokens[1].lower()\n",
    "                if word not in self.w2id:\n",
    "                    i = len(self.w2id)\n",
    "                    self.w2id[word]=i\n",
    "                    self.id2w[i]=word\n",
    "                if label not in self.l2id:\n",
    "                    self.l2id[label]=j\n",
    "                    self.id2l[j]=label\n",
    "                    j+=1\n",
    "                sent.append(self.w2id[word])\n",
    "                slab.append(self.l2id[label])\n",
    "        i=len(self.w2id)\n",
    "        self.w2id[\"<e>\"]=i\n",
    "        self.id2w[i]=\"<e>\"\n",
    "        print(\"[INFO] Loading Data ... Done\")\n",
    "        print(\"Vocabulary size: \",len(self.w2id),\n",
    "            len(self.id2w))\n",
    "        print(\"Number of sentences: \",len(self.sents))\n",
    "        print(\"Label size: \",len(self.l2id),len(self.id2l))\n",
    "        print(\"----------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Loading Embeddings ... Done\n",
      "Vocabulary size:  400000\n",
      "Embeddings:  [0.418, 0.24968, -0.41242, 0.1217, 0.34527, -0.044457, -0.49688, -0.17862, -0.00066023, -0.6566, 0.27843, -0.14767, -0.55677, 0.14658, -0.0095095, 0.011658, 0.10204, -0.12792, -0.8443, -0.12181, -0.016801, -0.33279, -0.1552, -0.23131, -0.19181, -1.8823, -0.76746, 0.099051, -0.42125, -0.19526, 4.0071, -0.18594, -0.52287, -0.31681, 0.00059213, 0.0074449, 0.17778, -0.15897, 0.012041, -0.054223, -0.29871, -0.15749, -0.34758, -0.045637, -0.44251, 0.18785, 0.0027849, -0.18411, -0.11514, -0.78581]\n",
      "----------\n",
      "[INFO] Loading Data ... Done\n",
      "Vocabulary size:  403139 403139\n",
      "Number of sentences:  2394\n",
      "Label size:  21 21\n",
      "----------\n"
     ]
    }
   ],
   "source": [
    "pp = preprocess(PATHglove,PATHner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BiLSTM_NER:\n",
    "    def __init__(self,pp,split=.9,batch_size=8,lr=1e-3):\n",
    "        self.pp = pp\n",
    "        self.split = split\n",
    "        self.batch_size = batch_size\n",
    "        self.lr = lr\n",
    "        self.prepareData()\n",
    "        self.prepareModel()\n",
    "        self.cce=CategoricalCrossentropy(reduction=\\\n",
    "            tf.keras.losses.Reduction.SUM)\n",
    "        self.opt=tf.keras.optimizers.Adam(learning_rate=lr)\n",
    "    def prepareData(self):\n",
    "        data=list(zip(pp.sents,pp.labels))\n",
    "        random.shuffle(data)\n",
    "        trainSet=data[:int(self.split*len(data))]\n",
    "        testSet=data[int(self.split*len(data)):]\n",
    "        self.trainX,self.trainY=list(zip(*trainSet))\n",
    "        self.testX,self.testY=list(zip(*testSet))\n",
    "        self.td = tf.data.Dataset.from_tensor_slices((\\\n",
    "                np.array(self.trainX),np.array(self.trainY)))\n",
    "        self.td = self.td.shuffle(buffer_size=1024)\\\n",
    "                .batch(self.batch_size)\n",
    "        self.vd = tf.data.Dataset.from_tensor_slices((\\\n",
    "                np.array(self.testX),np.array(self.testY)))\n",
    "        self.vd = self.vd.batch(self.batch_size)\n",
    "    def prepareModel(self):\n",
    "        embed_matrix = np.random.randn(self.pp.V,self.pp.k)\n",
    "        embed_matrix[:self.pp.V_,:]=np.array(self.pp.res)\n",
    "        x = Input(shape=(self.pp.maxLength,))\n",
    "        y = Embedding(self.pp.V,self.pp.k, \n",
    "            embeddings_initializer=tf.keras.initializers.Constant(\\\n",
    "            embed_matrix),trainable=True)(x)\n",
    "        z = Bidirectional(LSTM(self.pp.k,\n",
    "            return_sequences=True,dropout=0.2),merge_mode='concat')(y)\n",
    "        z_ = Dense(len(self.pp.l2id), activation=\"softmax\")(z)\n",
    "        self.model = Model(inputs=[x],outputs=[z_])\n",
    "    def train(self,epochs=500,printInterval=50):\n",
    "        trainLosses,testLosses=[],[]\n",
    "        lnum = len(self.pp.l2id)\n",
    "        for epoch in range(epochs):\n",
    "            total_train_loss=0\n",
    "            for i,(x,y) in enumerate(self.td):\n",
    "                with tf.GradientTape() as tape:\n",
    "                    pred=self.model(x,training=True)\n",
    "                    y = tf.one_hot(y,lnum)\n",
    "                    loss = self.cce(y,pred)\n",
    "                    total_train_loss+=loss.numpy()\n",
    "                grads = tape.gradient(loss,\n",
    "                        self.model.trainable_weights)\n",
    "                self.opt.apply_gradients(zip(grads,\n",
    "                        self.model.trainable_weights))\n",
    "\n",
    "            if epoch%printInterval==0:\n",
    "                total_val_loss=0\n",
    "                for j,(x,y) in enumerate(self.vd):\n",
    "                    pred=self.model(x,training=False)\n",
    "                    y = tf.one_hot(y,lnum)\n",
    "                    loss = self.cce(y,pred)\n",
    "                    total_val_loss+=loss\n",
    "                print(\"Epoch {} Train loss: {:.4f}\".format(\\\n",
    "                        epoch,float(total_train_loss/(i+1))))\n",
    "                print(\"Epoch {} Validation loss: {:.4f}\".format(\\\n",
    "                        epoch,float(total_val_loss/(j+1))))\n",
    "                trainLosses.append(total_train_loss/(i+1))\n",
    "                testLosses.append(total_val_loss/(j+1))\n",
    "        return trainLosses,testLosses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "nerModel = BiLSTM_NER(pp,batch_size=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 Train loss: 83430.6432\n",
      "Epoch 0 Validation loss: 22025.4453\n",
      "Epoch 50 Train loss: 4292.4626\n",
      "Epoch 50 Validation loss: 1761.0001\n",
      "Epoch 100 Train loss: 3168.4464\n",
      "Epoch 100 Validation loss: 1546.4680\n",
      "Epoch 150 Train loss: 2146.9435\n",
      "Epoch 150 Validation loss: 1446.1688\n",
      "Epoch 200 Train loss: 1472.3177\n",
      "Epoch 200 Validation loss: 1460.6355\n",
      "Epoch 250 Train loss: 1009.6657\n",
      "Epoch 250 Validation loss: 1529.0066\n",
      "Epoch 300 Train loss: 704.2049\n",
      "Epoch 300 Validation loss: 1611.2686\n",
      "Epoch 350 Train loss: 483.3982\n",
      "Epoch 350 Validation loss: 1710.8777\n",
      "Epoch 400 Train loss: 347.9608\n",
      "Epoch 400 Validation loss: 1796.9475\n",
      "Epoch 450 Train loss: 243.8927\n",
      "Epoch 450 Validation loss: 1862.5756\n"
     ]
    }
   ],
   "source": [
    "epochs,printInterval=500,50\n",
    "trainLosses,testLosses = nerModel.train(\\\n",
    "            epochs=epochs,printInterval=printInterval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'loss')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEGCAYAAACkQqisAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de3xcdZ3/8ddnJrcmnbTphSbQalupTEpbChQosruASGlhl6J4YQVh/alVV1xWVwX2t4qIu+LCCtYFXIS6IPxAF7xUKZQ71d9PLi2CFlroBbChLb2mbXpNMp/fH+ebNEknbTqZycwk7+fjMY+Z8z3fc853DmXe+Z7L95i7IyIikolYvhsgIiLFSyEiIiIZU4iIiEjGFCIiIpIxhYiIiGSsJN8N6GsjRozwsWPH5rsZIiJFY8mSJZvcfWS6eQMuRMaOHcvixYvz3QwRkaJhZm91N0+Hs0REJGMKERERyZhCREREMjbgzomISP/V3NxMQ0MDe/bsyXdTilJFRQWjR4+mtLS0x8soRESk32hoaCCRSDB27FjMLN/NKSruzubNm2loaGDcuHE9Xk6Hs0Sk39izZw/Dhw9XgGTAzBg+fPhh9+IUIiLSryhAMpfJvlOI9EBza4pbn17Jotc35rspIiIFRSHSAyUx4/ZFq3l46fp8N0VEClhjYyO33nprRsuee+65NDY29rj+N7/5TW688caMtpVNCpEeMDOStQmWrdue76aISAE7WIi0trYedNkFCxYwdOjQXDQrpxQiPZSsrea19TtIpfQkSBFJ76qrrmLVqlVMnTqVr371qzz99NOceeaZfPzjH2fy5MkAXHDBBZx44okce+yx3H777e3Ljh07lk2bNvHmm29SX1/PZz7zGY499lhmzJjB7t27D7rdl156ienTpzNlyhQ++MEPsnXrVgDmzp3LxIkTmTJlChdddBEAzzzzDFOnTmXq1Kkcf/zx7Nixo1ffWZf49lB9XYLdza38ecsuxo6oyndzROQQrv31K7y6NrtHDyYeWc01f3Nst/Ovv/56li5dyksvvQTA008/zfPPP8/SpUvbL5udN28ew4YNY/fu3Zx00klceOGFDB8+vNN6VqxYwX333cePfvQjPvrRj/Lggw9yySWXdLvdSy+9lB/84AecfvrpfOMb3+Daa6/l5ptv5vrrr+eNN96gvLy8/VDZjTfeyC233MJpp51GU1MTFRUVvdon6on0ULK2GoDl63VIS0R67uSTT+5038XcuXM57rjjmD59OmvWrGHFihUHLDNu3DimTp0KwIknnsibb77Z7fq3bdtGY2Mjp59+OgCXXXYZixYtAmDKlClcfPHF3HPPPZSURH2G0047jS9/+cvMnTuXxsbG9vJM5bQnYmZfAj4NOPAn4JNAHXA/MAx4EfiEu+8zs3LgbuBEYDPwMXd/M6znauBTQCvwD+6+MJTPBL4PxIE73P36XH2X945KYAbL1u1g5qS6XG1GRLLkYD2GvlRVtf/IxdNPP83jjz/O73//eyorKznjjDPS3pdRXl7e/jkejx/ycFZ3HnroIRYtWsT8+fO57rrreOWVV7jqqqs477zzWLBgAdOnT+fxxx8nmUxmtH7IYU/EzI4C/gGY5u6TiH7oLwK+C9zk7hOArUThQHjf6u5HAzeFepjZxLDcscBM4FYzi5tZHLgFmAVMBP421M2JQWVxxg2vUk9ERLqVSCQOeo5h27Zt1NTUUFlZyfLly3n22Wd7vc0hQ4ZQU1PDb3/7WwB+8pOfcPrpp5NKpVizZg1nnnkm//7v/05jYyNNTU2sWrWKyZMnc+WVVzJt2jSWL1/eq+3n+pxICTDIzJqBSmAd8H7g42H+XcA3gduA2eEzwAPAf1p058ts4H533wu8YWYrgZNDvZXuvhrAzO4PdV/N1Zepr6tm6dptuVq9iBS54cOHc9pppzFp0iRmzZrFeeed12n+zJkz+eEPf8iUKVM45phjmD59ela2e9ddd/G5z32OXbt2MX78eH784x/T2trKJZdcwrZt23B3vvSlLzF06FC+/vWv89RTTxGPx5k4cSKzZs3q1bbNPXdXG5nZFcC/AruBR4ErgGdDbwMzGwM87O6TzGwpMNPdG8K8VcApRMHyrLvfE8rvBB4Om5jp7p8O5Z8ATnH3y9O0Yw4wB+Bd73rXiW+91e3zVQ7qB0+s4D8ee51Xrj2HqnJdkyBSaJYtW0Z9fX2+m1HU0u1DM1vi7tPS1c/l4awaop7BOOBIoIro0FNXbSmW7n57z6D8wEL32919mrtPGzky7RMeeyRZF51cf+2d3l0SJyLSX+Ty6qwPAG+4+0Z3bwZ+DrwPGGpmbX/GjwbWhs8NwBiAMH8IsKVjeZdluivPmWRtAoDl6xQiIiKQ2xD5MzDdzCrDuY2ziM5XPAV8ONS5DPhV+Dw/TBPmP+nRsbb5wEVmVm5m44AJwPPAC8AEMxtnZmVEJ9/n5/D7MLpmEIPLS3RyXUQkyNmBfXd/zsweILqMtwX4A3A78BBwv5l9O5TdGRa5E/hJOHG+hSgUcPdXzOxnRAHUAnzB3VsBzOxyYCHRlV/z3P2VXH2fsD2StQn1REREgpyeHXb3a4BruhSvZv/VVR3r7gE+0s16/pXoBH3X8gXAgt63tOeSdQl+9dJa3F1DTovIgKc71g9TsraaHXtaWLtNj98UEVGIHKb6uraT6zovIiKd9WYoeICbb76ZXbt2pZ13xhlnsHjx4ozXnSsKkcP03lEhRNbrvIiIdJbLEClUCpHDlKgoZcywQXq2iIgcoOtQ8AA33HADJ510ElOmTOGaa6JTxDt37uS8887juOOOY9KkSfz0pz9l7ty5rF27ljPPPJMzzzzzoNu57777mDx5MpMmTeLKK68EoueV/N3f/R2TJk1i8uTJ3HTTTUD64eCzSbddZyBZW62eiEihe/gqWP+n7K6zdjLM6n6c165DwT/66KOsWLGC559/Hnfn/PPPZ9GiRWzcuJEjjzyShx56CIjG1BoyZAjf+973eOqppxgxYkS321i7di1XXnklS5YsoaamhhkzZvDLX/6SMWPG8Pbbb7N06VKA9qHf0w0Hn03qiWSgvjbB6o1N7Gk++JPKRGRge/TRR3n00Uc5/vjjOeGEE1i+fDkrVqxg8uTJPP7441x55ZX89re/ZciQIT1e5wsvvMAZZ5zByJEjKSkp4eKLL2bRokWMHz+e1atX88UvfpFHHnmE6upohI10w8Fnk3oiGUjWVZNyWLmhiUlH9fw/voj0oYP0GPqKu3P11Vfz2c9+9oB5S5YsYcGCBVx99dXMmDGDb3zjGz1eZzo1NTW8/PLLLFy4kFtuuYWf/exnzJs3L+1w8NkME/VEMtA2/InOi4hIR12Hgj/nnHOYN28eTU1NALz99tts2LCBtWvXUllZySWXXMJXvvIVXnzxxbTLp3PKKafwzDPPsGnTJlpbW7nvvvs4/fTT2bRpE6lUigsvvJDrrruOF198sdvh4LNJPZEMvHt4FRWlMZ0XEZFOug4Ff8MNN7Bs2TJOPfVUAAYPHsw999zDypUr+epXv0osFqO0tJTbbrsNgDlz5jBr1izq6up46qmn0m6jrq6O73znO5x55pm4O+eeey6zZ8/m5Zdf5pOf/CSpVAqA73znO90OB59NOR0KvhBNmzbNs3Gt9ez//B2DK0q499PZeR6AiPSehoLvvYIZCr6/S9ZWs2zdjm6PT4qIDAQKkQwl6xJs2bmPjU17890UEZG8UYhkKFkbXT6nEX1FCouODmQuk32nEMlQ+wOq9GwRkYJRUVHB5s2bFSQZcHc2b95MRUXFYS2nq7MyVFNVRm11hXoiIgVk9OjRNDQ0sHHjxnw3pShVVFQwevTow1pGIdILyboEr+peEZGCUVpayrhx4/LdjAFFh7N6IVlbzaqNTexrSeW7KSIieaEQ6YX6ugTNrc7qTdm9A1REpFgoRHqhvk5XaInIwKYQ6YVxI6ooi8dYpiu0RGSAUoj0Qmk8xtFHDFZPREQGLIVILyXrErpXREQGLIVIL9XXVvPO9r1s2bkv300REelzCpFeStbpznURGbgUIr2kMbREZCBTiPTSyEQ5IwaXqSciIgOSQiQLkrXVesqhiAxICpEsSNYmeG39DlpTGjlURAYWhUgWJOuq2duS4s3NO/PdFBGRPqUQyYL2Z4vo5LqIDDAKkSw4+ojBxGOmk+siMuAoRLKgojTO+BFVLFNPREQGGIVIliTrqtUTEZEBRyGSJcnaBA1bd7N9T3O+myIi0mcUIllSH4Y/eV33i4jIAKIQyZK24U+WKUREZABRiGRJ3ZAKqitKWL5O50VEZOBQiGSJmYWT6+qJiMjAoRDJovow/ElKw5+IyACR0xAxs6Fm9oCZLTezZWZ2qpkNM7PHzGxFeK8Jdc3M5prZSjP7o5md0GE9l4X6K8zssg7lJ5rZn8Iyc83Mcvl9DiVZV03T3hbebtydz2aIiPSZXPdEvg884u5J4DhgGXAV8IS7TwCeCNMAs4AJ4TUHuA3AzIYB1wCnACcD17QFT6gzp8NyM3P8fQ6qbfiTZTovIiIDRM5CxMyqgb8C7gRw933u3gjMBu4K1e4CLgifZwN3e+RZYKiZ1QHnAI+5+xZ33wo8BswM86rd/ffu7sDdHdaVF+8dlcAMnRcRkQEjlz2R8cBG4Mdm9gczu8PMqoBR7r4OILwfEeofBazpsHxDKDtYeUOa8gOY2RwzW2xmizdu3Nj7b9aNqvIS3j2sUneui8iAkcsQKQFOAG5z9+OBnew/dJVOuvMZnkH5gYXut7v7NHefNnLkyIO3upeStdUaQ0tEBoxchkgD0ODuz4XpB4hC5Z1wKIrwvqFD/TEdlh8NrD1E+eg05XlVX1fNm5t3smtfS76bIiKSczkLEXdfD6wxs2NC0VnAq8B8oO0Kq8uAX4XP84FLw1Va04Ft4XDXQmCGmdWEE+ozgIVh3g4zmx6uyrq0w7ryJlmXwB1ef6cp300REcm5khyv/4vAvWZWBqwGPkkUXD8zs08BfwY+EuouAM4FVgK7Ql3cfYuZXQe8EOp9y923hM+fB/4bGAQ8HF55VR+GP1m+bjtTxwzNc2tERHIrpyHi7i8B09LMOitNXQe+0M165gHz0pQvBib1splZNbpmEFVlcV2hJSIDgu5Yz7JYzDimNqF7RURkQFCI5EDbGFpR50pEpP9SiORAfW2CbbubWb99T76bIiKSUwqRHEjWtZ1c13kREenfFCI5cEzbGFq6c11E+jmFSA5UV5Ry1NBB6omISL+nEMmR+rqExtASkX5PIZIjydpqVm3cyd6W1nw3RUQkZxQiOZKsS9CaclZu0PAnItJ/KURyJFmrK7REpP9TiOTI2OGVlJfEdF5ERPo1hUiOlMRjvHdUQmNoiUi/phDJoWRtQg+oEpF+TSGSQ8m6ajY17WXjjr35boqISE4oRHKoPty5/poOaYlIP6UQyaG24U90cl1E+iuFSA4NH1zOEYlynRcRkX5LIZJj0bNF1BMRkf5JIZJj9bUJVrzTREtrKt9NERHJOoVIjiXrEuxrTfHGpp35boqISNYpRHKsbfiTZbpCS0T6IYVIjr1n5GBK48bydTovIiL9j0Ikx8pKYrxn5GCWKUREpB9SiPSB+rpqjaElIv2SQqQPJGsTrNu2h8Zd+/LdFBGRrFKI9IFkXXi2iHojItLPKET6QNsYWjq5LiL9jUKkD4xMlDOsqkw9ERHpdxQifcDMomeLKEREpJ9RiPSRZG01r6/fQWvK890UEZGsUYj0kWRdgt3Nrfx5y658N0VEJGt6FCJmdoWZVVvkTjN70cxm5Lpx/Ul9GP5EJ9dFpD/paU/kf7n7dmAGMBL4JHB9zlrVD00YNZiYaQwtEelfehoiFt7PBX7s7i93KJMeqCiNM25ElXoiItKv9DRElpjZo0QhstDMEoAekHGYkhr+RET6mZ6GyKeAq4CT3H0XUEp0SEsOQ31tgj9v2UXT3pZ8N0VEJCt6GiKnAq+5e6OZXQL8C7Atd83qn9qeLfKaeiMi0k/0NERuA3aZ2XHA14C3gLtz1qp+KlkXhj/RM9dFpJ/oaYi0uLsDs4Hvu/v3gURPFjSzuJn9wcx+E6bHmdlzZrbCzH5qZmWhvDxMrwzzx3ZYx9Wh/DUzO6dD+cxQttLMrurhd8mbo4YOIlFewvJ16omISP/Q0xDZYWZXA58AHjKzONF5kZ64AljWYfq7wE3uPgHYSnS+hfC+1d2PBm4K9TCzicBFwLHATODWEExx4BZgFjAR+NtQt2CZGcm6hHoiItJv9DREPgbsJbpfZD1wFHDDoRYys9HAecAdYdqA9wMPhCp3AReEz7PDNGH+WaH+bOB+d9/r7m8AK4GTw2ulu692933A/aFuQUvWVrN83Q6ijp2ISHHrUYiE4LgXGGJmfw3scfeenBO5megcStvlwMOBRndvuzypgSiQCO9rwvZaiE7cD+9Y3mWZ7soPYGZzzGyxmS3euHFjD5qdO8m6BDv2tvB24+68tkNEJBt6OuzJR4HngY8AHwWeM7MPH2KZvwY2uPuSjsVpqvoh5h1u+YGF7re7+zR3nzZy5MiDtDr3ku3Dn+i8iIgUv5Ie1vvfRPeIbAAws5HA4+w/LJXOacD5ZnYuUAFUE/VMhppZSehtjAbWhvoNwBigwcxKgCHAlg7lbTou0115wTqmdv8VWh+YOCrPrRER6Z2enhOJtQVIsPlQy7r71e4+2t3HEp0Yf9LdLwaeAtp6MZcBvwqf54dpwvwnwxVh84GLwtVb44AJRL2iF4AJ4WqvsrCN+T38PnkzuLyEdw2r1BhaItIv9LQn8oiZLQTuC9MfAxZkuM0rgfvN7NvAH4A7Q/mdwE/MbCVRD+QiAHd/xcx+BrwKtABfcPdWADO7HFgIxIF57v5Khm3qU8nahMbQEpF+wXp6lZCZXUh0iMqARe7+i1w2LFemTZvmixcvzmsbbnrsdX7w5Ape/dZMKkrjeW2LiMihmNkSd5+Wbl5PeyK4+4PAg1lr1QBWX5cg5bDinSYmjx6S7+aIiGTsoCFiZjtIf8WTAe7u1TlpVT/XdoXWsvXbFSIiUtQOGiLu3qOhTeTwvGtYJYNK47rMV0SKnp6xngexmHFMbYJlOrkuIkVOIZIn9WEMLQ1/IiLFTCGSJ8naarbuambDjr35boqISMYUInmSDHeu65CWiBQzhUietI+hpTvXRaSIKUTyZEhlKUcOqdCd6yJS1BQieZSsq1ZPRESKmkIkj5K1CVZuaGJfS+rQlUVECpBCJI+SddW0pJxVG5vy3RQRkYwoRPKovsOzRUREipFCJI/GjaiiLB7T8CciUrQUInlUEo8xYdRgPaBKRIqWQiTPkrXVusxXRIqWQiTP6usSbNixl81NGv5ERIqPQiTP2u5cf02HtESkCClE8ixZF8bQUoiISBFSiOTZiMHljBhcrvMiIlKUFCIFIHq2iHoiIlJ8FCIFIFmb4PV3dtDSquFPRKS4KEQKQLK2mr0tKd7cvCvfTREROSwKkQLQdnJdw5+ISLFRiBSAo48YTEnMNPyJiBQdhUgBKC+J856Rg9UTEZGioxApEMm6BMvUExGRIqMQKRDJ2mrebtzN9j3N+W6KiEiPKUQKRNvJdQ1/IiLFRCFSIOrDGFq6c11EiolCpECMqi5naGWpxtASkaKiECkQZkayNsEy9UREpIgoRApIsraa19bvIJXyfDdFRKRHFCIFpL4uwa59razZquFPRKQ4KEQKSNsDqnS/iIgUC4VIAXnvqARmGkNLRIqHQqQnmvfAI/8Mq5/J6WYGlcUZN7xKY2iJSNFQiPSEp2DFQvjl52H31pxuKlmXUE9ERIqGQqQnyirhQz+CpnfgoX/K6aaStdW8tWUXO/e25HQ7IiLZkLMQMbMxZvaUmS0zs1fM7IpQPszMHjOzFeG9JpSbmc01s5Vm9kczO6HDui4L9VeY2WUdyk80sz+FZeaameXq+3DUCXDGVbD0Qfjj/+RsM8naBO7w+js6pCUihS+XPZEW4J/cvR6YDnzBzCYCVwFPuPsE4IkwDTALmBBec4DbIAod4BrgFOBk4Jq24Al15nRYbmYOvw/8xZdhzPSoN9L455xsor4uDH+iO9dFpAjkLETcfZ27vxg+7wCWAUcBs4G7QrW7gAvC59nA3R55FhhqZnXAOcBj7r7F3bcCjwEzw7xqd/+9uztwd4d15UYsDh/6r+gcyS8+D6nWrG/iqKGDGFxeojG0RKQo9Mk5ETMbCxwPPAeMcvd1EAUNcESodhSwpsNiDaHsYOUNacrTbX+OmS02s8UbN27s3ZepGQuzvgtv/Q7+3w96t640YjHjmNqExtASkaKQ8xAxs8HAg8A/uvvB/rxOdz7DMyg/sND9dnef5u7TRo4ceagmH9rUj0P9+fDkt2Hdy71fXxfJ2gTL120n6mCJiBSunIaImZUSBci97v7zUPxOOBRFeN8QyhuAMR0WHw2sPUT56DTluWcGf/N9qBwOP58DzbuzuvpkXTXb97SwbtuerK5XRCTbcnl1lgF3Asvc/XsdZs0H2q6wugz4VYfyS8NVWtOBbeFw10JghpnVhBPqM4CFYd4OM5setnVph3XlXuUwuOBW2LgcHv9mVlddXxs9oEr3i4hIoctlT+Q04BPA+83spfA6F7geONvMVgBnh2mABcBqYCXwI+DvAdx9C3Ad8EJ4fSuUAXweuCMsswp4OIff50BHnwWnfA6e+yGsfDxrq31vCBGNoSUiha4kVyt299+R/rwFwFlp6jvwhW7WNQ+Yl6Z8MTCpF83svQ98E1Y/Db/8e/j876FqeK9XWV1RyuiaQbrMV0QKnu5Y763SQdHd7Lu2wG+ugCydDE/WVusyXxEpeAqRbKibAmd9HZb9Gl76P1lZ5cS6BKs37WRPc/bvRRERyRaFSLacejmM/Ut4+Guw5Y1ery5ZV01rylm5oSkLjRMRyQ2FSLbE4nDBbWBx+MVnobV3Aygm26/Q0nkRESlcCpFsGjoGzvsPWPMc/O6mXq3q3cOrqCiN6byIiBQ0hUi2TfkITPowPHM9vL0k49XEY8YxoxLqiYhIQVOI5MJ5N8Lg2uhu9n07M15NsrZaNxyKSEFTiOTCoBr44G2weRU8+i8ZryZZl2BT0z427tibxcaJiGSPQiRXxv0VvO9yWDwPXnsko1Uka9ueLaLeiIgUJoVILr3/6zBqEsy/HJoOfwj69iu0NPyJiBQohUgulZRHd7Pv2Q7zv3jYd7PXVJVRW13BMvVERKRAKURybdREOPtaeP1hWPLfh714si6hgRhFpGApRPrCyZ+F8WfAwn+GTSsPa9FkbTUrN+yguTWVk6aJiPSGQqQvxGLR3ezxMvj5Z6C1uceL1tclaG51Vm/M/FJhEZFcUYj0leojo6chrn0RFt3Q48V0hZaIFDKFSF869gI47uNRiKx5vkeLjB9ZRWncdF5ERAqSQqSvzfouDBkdHdbae+hgKI3HOPqIhHoiIlKQFCJ9raIaPng7NP4ZHrmqR4vU1yZ0r4iIFCSFSD68+1T4iy/BH+6BV+cfsnqyLsH67XvYunNfHzRORKTnFCL5cvpVUDcVfn0F7Fh/0Kr7T66rNyIihUUhki8lZdHd7M274Zd/f9C72ZN1bQ+o0nkRESksCpF8GvlemHEdrHoCnv9R99UGlzO8qkznRUSk4ChE8u2kT8PRZ8NjX4cNy9NWMTOSdbpCS0QKj0Ik38xg9i1QVgU//zS0pD95nqyt5rV3dtCaOrxBHEVEckkhUggSo+D8H8D6P8HT/5a2SrI2wZ7mFG9t1vAnIlI4FCKFInkenHAZ/O5mePP/HjC7vk5XaIlI4VGIFJJz/g2GjYNffBb2bOs06+gjBhOPGcvX6byIiBQOhUghKR8c3c2+fS0s+GqnWRWlccaPqGKZeiIiUkAUIoVmzElw+tfgjz+FpQ92mpWsq9YVWiJSUBQihegvvwJHTYPffAm2vd1enKxNsGbLbnbs6fnzSEREckkhUojiJfCh26G1BX75OUhFTzWsD3eu3/HbN3j0lfUsfnMLqzY2sXXnPlK69FdE8qAk3w2Qbgx/D8z8Dvz6H+DZW+F9lzNl9FCqyuJ8/4kVB1SPGdRUllFTVcawyjKGVYXPVaXUhOm2V9t0ZVkcM8vDlxOR/kIhUshOuBReXwhPXAvjz2BE7SSWfP1sNu/cx9ad+9iycx9bd+1jc1P03nF69aYmtrzVzNZd+7q9QbG8JNYpVGqqyhjePl26P5AGR+9DK8soK1HnVUT2Mz/IwH/90bRp03zx4sX5bkbP7dwEt54KVSPhM09CacVhLZ5KOTv2tLClLWR27uv8uS14Okxv39PS7foS5SUMGxwFSk1lKUMHlTK0soyh4XNNVRlDBkW9n6isjERFCbGYejwixcrMlrj7tHTz1BMpdFUj4IJb4d4Pw5PXwTn/eliLx2LGkMpShlSWMm5EVY+WaW5N0birmS0dQqYtdDZ3mN6ycx+rNjbRuKuZHQcJnpjBkK5hU1nGkBAyNVWl7fNrQtmQylKqK0p0uE2kwClEisGEs6OBGn//n9Hn8WfkdHOl8RgjE+WMTJT3eJmW1hTbdjfTuLuZxl37aNzVzNZd0edtu6PDao27mtm2u5mNTXtZsSEKn6a93YdPPGYhXLr2eKL3mspSqgeVUlVWQmV5nKqyEqrK41SVl1BZVkJVWZySuA6/ieSSQqRYnH0drH4GfvF5OO0KiMUhVtLlFesy3aWOxdMsF09TN91yB/8xLonHGD64nOGDex48EPV6th0ieNqC6Z3te3ht/Q4ad+1j577WHq2/rCRGVVk8CpUQMFVlJVSWtYVNh/eykmh+ebw9hCrLO79XlZdQOlCCyT28UkB4P+h0eLXPS3Wu0/EFB5YdUK/rtrqr24N6Hdt3QN3uvhMHWX/XMg69nzp9Jk1Z1/3Xtcy7KUsdWJZuHRVD4YJbsv7PRCFSLMoq4cIfwV3nwyNX5qEBdmBgWZoX1mHaurwfWK/UjBEWY8RB6xmUxmBoDGqieimM5hS0pKA15bSknNaU09qaotXD51QqvEfzU80pWvd6e1n7fHdSKcc6fVunGdgWXsb+c4dmRkks6inFzfX5xlcAAAkySURBVIjFjBKDeMyJAfFYdO18zCBuTswgZk6c6D1mbfOjbcbMibH/s0E0bY6Fcuv2h4Y0ZYfzQ3+QutILtv/fc7rPB5R1s0yn91iasnT1ullH5fCcfNOiDxEzmwl8H4gDd7j79XluUu4ceTx8ZQU074JUS4dXa3h1KfM0ZQd87lLHU12WSVe3dX/dtH/9cWBZ2r9O0/2FmuavvVQKvLlTvRhOuaco9xR0/PmPfomjfw3tBR0ccI4l+p/MgZQ7rR5trv1zCJiUW5jntHoqvBMCKNRvMVoc9jphnoX36Kukwm1ZbdHgRGUeykhTFr3HMDNisRixmBGzGBaLEbMwHYu1z7NYDItFl2631YneDWv7HIth7cvGsfAeMyMWj0frjYf1x6PyeHz//Hj79mKUxOP7t23x/T9asXR/VHT94yLW5Qevu3pd5nVbt7t6XbfR9Ye2uzodA8B6uN62H/qBo6hDxMziwC3A2UAD8IKZzXf3V/PbshwqrTjsK7Tk0Iwod+KHqpgB96gntKe5lb0tKfa2pKLPzSn2trSyJ7y3l4c6e9s+N7eyp8P0Aetp2b+e5pbwSjnNrSlaWp19rSmaW1MHewJzVpTGjdJ4jJJY9B6PGSWx0FOLWZgO5XFrn9+pvG06bsRjsQ7zO77H0iwflcdjdFouHjNiFvUeo14jURjaoeeZhb9HYm09yFTUo4wZZq2d1nHgcvs/x8M8C9vqWN/aP9OpTjEp6hABTgZWuvtqADO7H5gN9N8QkaJjZu0/sIk8tqM1BEtbuDS3hrBpSdGSSrGvxWlJpUKdziHUXj/Mi+qnaAnL7w+t/fNbU05Lq3c63BiVQ2sq1aHM2d3cGqZT7cscuFyYbu1c3h8Ha+guaCxt8HQMpu7qwPCqcn72uVOz3tZiD5GjgDUdphuAU7pWMrM5wByAd73rXX3TMpECE/1lHqeiNBf9rfxJhfNanUMmCp1U+yFHx9s+tx+iJNRJP69TvUPM87D9tMulovlt8yDUCdMeDpO2T0Oon76OE6Y71PEOh1UPrBO9J8pz83Nf7CGSrt93wN8l7n47cDtENxvmulEi0ndiMSOG0c+ysWgU+7WKDcCYDtOjgbV5aouIyIBT7CHyAjDBzMaZWRlwETA/z20SERkwivpwlru3mNnlwEKiC2vmufsreW6WiMiAUdQhAuDuC4AF+W6HiMhAVOyHs0REJI8UIiIikjGFiIiIZEwhIiIiGRtwTzY0s43AWxkuPgLYlMXmFDPti860PzrT/tivP+yLd7v7yHQzBlyI9IaZLe7uEZEDjfZFZ9ofnWl/7Nff94UOZ4mISMYUIiIikjGFyOG5Pd8NKCDaF51pf3Sm/bFfv94XOiciIiIZU09EREQyphAREZGMKUR6wMxmmtlrZrbSzK7Kd3v6gpnNM7MNZra0Q9kwM3vMzFaE95pQbmY2N+yfP5rZCflrefaZ2Rgze8rMlpnZK2Z2RSgfqPujwsyeN7OXw/64NpSPM7Pnwv74aXg8A2ZWHqZXhvlj89n+XDCzuJn9wcx+E6YHzL5QiByCmcWBW4BZwETgb81sYn5b1Sf+G5jZpewq4Al3nwA8EaYh2jcTwmsOcFsftbGvtAD/5O71wHTgC+HfwEDdH3uB97v7ccBUYKaZTQe+C9wU9sdW4FOh/qeAre5+NHBTqNffXAEs6zA9YPaFQuTQTgZWuvtqd98H3A/MznObcs7dFwFbuhTPBu4Kn+8CLuhQfrdHngWGmlld37Q099x9nbu/GD7vIPqxOIqBuz/c3ZvCZGl4OfB+4IFQ3nV/tO2nB4CzzCzdo62LkpmNBs4D7gjTxgDaFwqRQzsKWNNhuiGUDUSj3H0dRD+swBGhfMDso3D44XjgOQbw/giHb14CNgCPAauARndvCVU6fuf2/RHmbwOG922Lc+pm4GtAKkwPZwDtC4XIoaX7K0HXRXc2IPaRmQ0GHgT+0d23H6xqmrJ+tT/cvdXdpwKjiXrr9emqhfd+uz/M7K+BDe6+pGNxmqr9dl8oRA6tARjTYXo0sDZPbcm3d9oOy4T3DaG83+8jMyslCpB73f3noXjA7o827t4IPE10rmiombU9LbXjd27fH2H+EA48VFqsTgPON7M3iQ51v5+oZzJg9oVC5NBeACaEqy3KgIuA+XluU77MBy4Lny8DftWh/NJwVdJ0YFvbYZ7+IByzvhNY5u7f6zBroO6PkWY2NHweBHyA6DzRU8CHQ7Wu+6NtP30YeNL7yV3O7n61u49297FEvw1PuvvFDKR94e56HeIFnAu8TnTc93/nuz199J3vA9YBzUR/PX2K6NjtE8CK8D4s1DWiK9hWAX8CpuW7/VneF39BdMjhj8BL4XXuAN4fU4A/hP2xFPhGKB8PPA+sBP4HKA/lFWF6ZZg/Pt/fIUf75QzgNwNtX2jYExERyZgOZ4mISMYUIiIikjGFiIiIZEwhIiIiGVOIiIhIxhQiIgXOzM5oGx1WpNAoREREJGMKEZEsMbNLwnM2XjKz/wqDFDaZ2X+Y2Ytm9oSZjQx1p5rZs+F5I7/o8CySo83s8fCsjhfN7D1h9YPN7AEzW25m97aN/Gpm15vZq2E9N+bpq8sAphARyQIzqwc+Bpzm0cCErcDFQBXworufADwDXBMWuRu40t2nEN3V3lZ+L3CLR8/qeB/RqAEQjRz8j0TPtBkPnGZmw4APAseG9Xw7t99S5EAKEZHsOAs4EXghDJF+FtGPfQr4aahzD/AXZjYEGOruz4Tyu4C/MrMEcJS7/wLA3fe4+65Q53l3b3D3FNGwK2OB7cAe4A4z+xDQVlekzyhERLLDgLvcfWp4HePu30xT72DjDB3s4UR7O3xuBUo8eh7FyUSjC18APHKYbRbpNYWISHY8AXzYzI6A9uevv5vo/7G20Vw/DvzO3bcBW83sL0P5J4BnPHpGSYOZXRDWUW5mld1tMDzfZIi7LyA61DU1F19M5GBKDl1FRA7F3V81s38BHjWzGNHox18AdgLHmtkSoqfYfSwschnwwxASq4FPhvJPAP9lZt8K6/jIQTabAH5lZhVEvZgvZflriRySRvEVySEza3L3wfluh0iu6HCWiIhkTD0RERHJmHoiIiKSMYWIiIhkTCEiIiIZU4iIiEjGFCIiIpKx/w+Z3JMFJyvMUQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(range(0,epochs,printInterval),\n",
    "         trainLosses,label='train loss')\n",
    "plt.plot(range(0,epochs,printInterval),\n",
    "         testLosses,label='test loss')\n",
    "plt.legend()\n",
    "plt.xlabel('epochs')\n",
    "plt.ylabel('loss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 0 0 0 0 0 0 0 6 0 0 0]\n",
      "[0 0 0 0 0 0 0 0 0 6 0 0 0]\n",
      "[0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0]\n",
      "[0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0]\n",
      "[0 0 5 0 0 0 0 0 0 0 0 0 0]\n",
      "[0 0 5 0 0 0 0 0 0 0 0 0 0]\n",
      "[0 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      "[0 0 0 0 0 0 0 0 0 0 3 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      "[ 0  0  0  0  5  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0 14  0\n",
      "  0]\n",
      "[ 0  0  0  0  5  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0  0 14  0\n",
      "  0]\n",
      "[10  0  0  0  0  0 10 18  0 10  0  0 10  0  0  0  0  0]\n",
      "[10  0  0  0  0  0 10 18  0 10  0  0 10  0  0  0  0  0]\n",
      "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n"
     ]
    }
   ],
   "source": [
    "show = 10\n",
    "for i in range(show):\n",
    "    x,y=nerModel.trainX[i],nerModel.trainY[i]\n",
    "    ind = np.where(x==pp.w2id[\"<e>\"])[0][0]\n",
    "    pred = nerModel.model(x.reshape(1,-1),training=False)\n",
    "    pred = tf.math.argmax(pred,2).numpy()[0][:ind]\n",
    "    y = y[:ind]\n",
    "    print(pred)\n",
    "    print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculateMetrics(dataset,name):\n",
    "    up,down=0,0\n",
    "    TPs,FPs,FNs=dict(),dict(),dict()\n",
    "    for i,(x,y) in enumerate(dataset):\n",
    "        pred=nerModel.model(x,training=False)\n",
    "        y = y.numpy()\n",
    "        pred=tf.math.argmax(pred,2).numpy()\n",
    "        for k in range(len(x)):\n",
    "            ind = np.where(x[k]==pp.w2id[\"<e>\"])[0][0]\n",
    "            up+=(y[k][:ind]==pred[k][:ind]).sum()\n",
    "            down+=np.prod(pred[k][:ind].shape)\n",
    "            for e in range(ind):\n",
    "                if pred[k][e]==y[k][e]:\n",
    "                    TPs[y[k][e]]=TPs.get(y[k][e],0)+1\n",
    "                elif pred[k][e]!=y[k][e]:\n",
    "                    FPs[pred[k][e]]=FPs.get(pred[k][e],0)+1\n",
    "                    FNs[y[k][e]]=FNs.get(y[k][e],0)+1\n",
    "    print(\"{} accuracy: {:.2f}\".format(name,up/down))\n",
    "    for key in pp.id2l:\n",
    "        f1 = TPs.get(key,0)/(TPs.get(key,0)+\\\n",
    "                FPs.get(key,0)*.5+FNs.get(key,0)*.5)\n",
    "        print(\"Label {} F1 score : {:.2f}\".format(pp.id2l[key],f1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training accuracy: 1.00\n",
      "Label o F1 score : 1.00\n",
      "Label b-facility F1 score : 0.98\n",
      "Label i-facility F1 score : 0.98\n",
      "Label b-other F1 score : 0.98\n",
      "Label i-other F1 score : 0.99\n",
      "Label b-company F1 score : 0.98\n",
      "Label b-person F1 score : 1.00\n",
      "Label b-tvshow F1 score : 0.97\n",
      "Label b-sportsteam F1 score : 0.97\n",
      "Label i-person F1 score : 1.00\n",
      "Label b-geo-loc F1 score : 0.99\n",
      "Label b-movie F1 score : 0.98\n",
      "Label i-movie F1 score : 0.97\n",
      "Label i-tvshow F1 score : 0.96\n",
      "Label b-product F1 score : 0.98\n",
      "Label i-company F1 score : 0.97\n",
      "Label b-musicartist F1 score : 0.98\n",
      "Label i-musicartist F1 score : 0.97\n",
      "Label i-geo-loc F1 score : 0.99\n",
      "Label i-product F1 score : 0.99\n",
      "Label i-sportsteam F1 score : 0.97\n",
      "----------\n",
      "Testing accuracy: 0.95\n",
      "Label o F1 score : 0.97\n",
      "Label b-facility F1 score : 0.25\n",
      "Label i-facility F1 score : 0.24\n",
      "Label b-other F1 score : 0.45\n",
      "Label i-other F1 score : 0.45\n",
      "Label b-company F1 score : 0.54\n",
      "Label b-person F1 score : 0.36\n",
      "Label b-tvshow F1 score : 0.00\n",
      "Label b-sportsteam F1 score : 0.35\n",
      "Label i-person F1 score : 0.20\n",
      "Label b-geo-loc F1 score : 0.23\n",
      "Label b-movie F1 score : 0.00\n",
      "Label i-movie F1 score : 0.00\n",
      "Label i-tvshow F1 score : 0.00\n",
      "Label b-product F1 score : 0.33\n",
      "Label i-company F1 score : 0.00\n",
      "Label b-musicartist F1 score : 0.22\n",
      "Label i-musicartist F1 score : 0.00\n",
      "Label i-geo-loc F1 score : 0.25\n",
      "Label i-product F1 score : 0.00\n",
      "Label i-sportsteam F1 score : 0.33\n"
     ]
    }
   ],
   "source": [
    "calculateMetrics(nerModel.td,\"Training\")\n",
    "print(\"----------\")\n",
    "calculateMetrics(nerModel.vd,\"Testing\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Notes\n",
    "- It could be more useful to calculate F1 score for each class."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}