create_results_tables.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "/data/anaconda/envs/py37torch/lib/python3.7/site-packages/tqdm/std.py:668: FutureWarning: The Panel class is removed from pandas. Accessing it from the top-level namespace will also be removed in the next version\n  from pandas import Panel\n"
     ]
    }
   ],
   "source": [
    "import scipy\n",
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "from utils.classificaton_utils import evaluate_results\n",
    "\n",
    "avg = \"none\" # either macro or none (if none, we consider label 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "renamed_columns = {\"def_tok_overlap_ranking\": \"Lesk: token overlap\",\n",
    "                   \"sent_embedding\": \"Lesk: sentence embedding\",\n",
    "                   \"w2v_lesk_ranking\": \"Lesk: w2v\",\n",
    "                   \"svm_wemb_baseline\": \"SVM classifier\",\n",
    "                   \"bert_binary_centroid_vector_bert_base_-1,-2,-3,-4_mean\": \"BERT_base binary centroid\",\n",
    "                   \"bert_centroid_sense_vector_bert_base_-1,-2,-3,-4_mean\": \"BERT_base sense centroid\",\n",
    "                   \"bert_ml_perceptron_vector_bert_base_-1,-2,-3,-4_mean\": \"BERT_base perceptron\",\n",
    "                   \"bert_binary_centroid_vector_blert_base_-1,-2,-3,-4_mean\": \"BERT_1900 binary centroid\",\n",
    "                   \"bert_centroid_sense_vector_blert_base_-1,-2,-3,-4_mean\": \"BERT_1900 sense centroid\",\n",
    "                   \"bert_ml_perceptron_vector_blert_base_-1,-2,-3,-4_mean\": \"BERT_1900 perceptron\",\n",
    "                   \"bert_binary_centroid_vector_bert_1850_-1,-2,-3,-4_mean\": \"BERT_1850 binary centroid\",\n",
    "                   \"bert_centroid_sense_vector_bert_1850_-1,-2,-3,-4_mean\": \"BERT_1850 sense centroid\",\n",
    "                   \"bert_ml_perceptron_vector_bert_1850_-1,-2,-3,-4_mean\": \"BERT_1850 perceptron\",\n",
    "                   \"bert_ts_nearest_centroid_sense_vector_bert_base_-1,-2,-3,-4_mean\": \"BERT_base nearest sense centroid\",\n",
    "                   \"bert_ts_weighted_centroid_sense_vector_bert_base_-1,-2,-3,-4_mean\": \"BERT_base weighted sense centroid\",\n",
    "                   \"bert_ts_nearest_centroid_sense_vector_blert_base_-1,-2,-3,-4_mean\": \"BERT_1900 nearest sense centroid\",\n",
    "                   \"bert_ts_weighted_centroid_sense_vector_blert_base_-1,-2,-3,-4_mean\": \"BERT_1900 weighted sense centroid\",\n",
    "                   \"bert_ts_nearest_centroid_sense_vector_bert_1850_-1,-2,-3,-4_mean\": \"BERT_1850 nearest sense centroid\",\n",
    "                   \"bert_ts_weighted_centroid_sense_vector_bert_1850_-1,-2,-3,-4_mean\": \"BERT_1850 weighted sense centroid\"}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Table 2: Main evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_1850 = evaluate_results(Path(f'results_1850'),avg=avg)\n",
    "results_1920 = evaluate_results(Path(f'results_1920'),avg=avg)\n",
    "results_2000 = evaluate_results(Path(f'results_2000'),avg=avg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_1850 = pd.DataFrame.from_dict(results_1850, orient='index', columns=['precision','recall','fscore','preds'])\n",
    "df_1920 = pd.DataFrame.from_dict(results_1920, orient='index', columns=['precision','recall','fscore','preds'])\n",
    "df_2000 = pd.DataFrame.from_dict(results_2000, orient='index', columns=['precision','recall','fscore','preds'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "df_1850 = df_1850.rename(index=renamed_columns)\n",
    "df_1920 = df_1920.rename(index=renamed_columns)\n",
    "df_2000 = df_2000.rename(index=renamed_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fscores = pd.concat([df_1850[['precision', 'recall', 'fscore']], df_1920[['precision', 'recall', 'fscore']], df_2000[['precision', 'recall', 'fscore']]],axis=1) # ,df_2000['fscore']\n",
    "cols_baselines = [c for c in df_fscores.index if not c.startswith(\"bert_\")]\n",
    "cols_bert = [c for c in df_fscores.index if c.startswith(\"bert_\") and not 'ts' in c and not 'contrast' in c]\n",
    "df_fscores = df_fscores.loc[cols_baselines + cols_bert]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\\begin{tabular}{lrrrrrrrrr}\n\\toprule\n{} &  precision &  recall &  fscore &  precision &  recall &  fscore &  precision &  recall &  fscore \\\\\n\\midrule\nrandom                    &      0.102 &   0.511 &   0.170 &      0.087 &   0.483 &   0.148 &      0.087 &   0.503 &   0.148 \\\\\nLesk: token overlap       &      0.234 &   0.266 &   0.249 &      0.245 &   0.278 &   0.261 &      0.248 &   0.277 &   0.261 \\\\\nLesk: sentence embedding  &      0.269 &   0.196 &   0.227 &      0.266 &   0.198 &   0.227 &      0.280 &   0.215 &   0.243 \\\\\nLesk: w2v                 &      0.323 &   0.291 &   0.306 &      0.288 &   0.270 &   0.279 &      0.286 &   0.257 &   0.271 \\\\\nSVM classifier            &      0.500 &   0.091 &   0.155 &      0.495 &   0.083 &   0.143 &      0.509 &   0.077 &   0.133 \\\\\nBERT\\_base binary centroid &      0.254 &   0.699 &   0.373 &      0.238 &   0.702 &   0.356 &      0.236 &   0.716 &   0.355 \\\\\nBERT\\_base sense centroid  &      0.756 &   0.464 &   0.575 &      0.665 &   0.471 &   0.552 &      0.618 &   0.493 &   0.548 \\\\\nBERT\\_base perceptron      &      0.578 &   0.425 &   0.490 &      0.575 &   0.448 &   0.504 &      0.580 &   0.456 &   0.510 \\\\\nBERT\\_1900 binary centroid &      0.234 &   0.698 &   0.351 &      0.221 &   0.715 &   0.338 &      0.222 &   0.728 &   0.340 \\\\\nBERT\\_1900 sense centroid  &      0.766 &   0.498 &   0.604 &      0.702 &   0.512 &   0.592 &      0.630 &   0.497 &   0.556 \\\\\nBERT\\_1900 perceptron      &      0.575 &   0.429 &   0.492 &      0.588 &   0.453 &   0.511 &      0.586 &   0.463 &   0.517 \\\\\nBERT\\_1850 binary centroid &      0.229 &   0.678 &   0.343 &      0.224 &   0.713 &   0.340 &      0.222 &   0.722 &   0.339 \\\\\nBERT\\_1850 sense centroid  &      0.789 &   0.486 &   0.602 &      0.688 &   0.500 &   0.579 &      0.613 &   0.495 &   0.548 \\\\\nBERT\\_1850 perceptron      &      0.587 &   0.424 &   0.492 &      0.568 &   0.437 &   0.494 &      0.570 &   0.456 &   0.506 \\\\\n\\bottomrule\n\\end{tabular}\n\n"
     ]
    }
   ],
   "source": [
    "print(df_fscores.to_latex())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "rows_1850 = [r for r in df_fscores.index if '1850' in r]\n",
    "rows_1920 = [r for r in df_fscores.index if '1900' in r]\n",
    "rows_2000 = [r for r in df_fscores.index if 'base' in r]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "diff_1850_2000 = df_1850.loc[df_1850.index.isin(rows_1850)].fscore.values - df_1850.loc[df_1850.index.isin(rows_2000)].fscore.values\n",
    "diff_1920_2000 = df_1920.loc[df_1920.index.isin(rows_1920)].fscore.values - df_1920.loc[df_1920.index.isin(rows_2000)].fscore.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\\begin{tabular}{lrr}\n\\toprule\n{} &      0 &      1 \\\\\n\\midrule\nBERT\\_base binary centroid & -0.010 & -0.011 \\\\\nBERT\\_base sense centroid  &  0.011 &  0.025 \\\\\nBERT\\_base perceptron      & -0.009 &  0.001 \\\\\n\\bottomrule\n\\end{tabular}\n\n"
     ]
    }
   ],
   "source": [
    "df_diff = pd.DataFrame([diff_1850_2000,diff_1920_2000],columns=rows_2000).T\n",
    "#df_diff['sum'] = df_diff.sum(axis=1)\n",
    "print(df_diff.to_latex())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Table 3: Time-sensitive methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_ts_1850 = evaluate_results(Path(\"results_ts_1850\"),avg=avg)\n",
    "results_ts_1920 = evaluate_results(Path(\"results_ts_1920\"),avg=avg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_ts_1850 = pd.DataFrame.from_dict(results_ts_1850, orient='index', columns=['precision','recall','fscore','preds'])\n",
    "df_ts_1920 = pd.DataFrame.from_dict(results_ts_1920, orient='index', columns=['precision','recall','fscore','preds'])\n",
    "df_fscores = pd.concat([df_ts_1850['fscore'], df_ts_1920['fscore']],axis=1)\n",
    "df_fscores = df_fscores.rename(index=renamed_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "Index(['BERT_base sense centroid', 'BERT_base nearest sense centroid',\n",
       "       'BERT_base weighted sense centroid', 'BERT_1900 sense centroid',\n",
       "       'BERT_1900 nearest sense centroid', 'BERT_1900 weighted sense centroid',\n",
       "       'BERT_1850 sense centroid', 'BERT_1850 nearest sense centroid',\n",
       "       'BERT_1850 weighted sense centroid'],\n",
       "      dtype='object')"
      ]
     },
     "metadata": {},
     "execution_count": 11
    }
   ],
   "source": [
    "df_fscores.index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\\begin{tabular}{lrr}\n\\toprule\n{} &  fscore &  fscore \\\\\n\\midrule\nBERT\\_base sense centroid          &   0.575 &   0.552 \\\\\nBERT\\_base nearest sense centroid  &   0.458 &   0.433 \\\\\nBERT\\_base weighted sense centroid &   0.593 &   0.556 \\\\\nBERT\\_1900 sense centroid          &   0.604 &   0.592 \\\\\nBERT\\_1900 nearest sense centroid  &   0.505 &   0.464 \\\\\nBERT\\_1900 weighted sense centroid &   0.627 &   0.584 \\\\\nBERT\\_1850 sense centroid          &   0.602 &   0.579 \\\\\nBERT\\_1850 nearest sense centroid  &   0.489 &   0.441 \\\\\nBERT\\_1850 weighted sense centroid &   0.609 &   0.562 \\\\\n\\bottomrule\n\\end{tabular}\n\n"
     ]
    }
   ],
   "source": [
    "print(df_fscores.to_latex())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Curated examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_curated_seed = evaluate_results(Path(f'results_curated_1920_seed'),avg=avg)\n",
    "results_curated_synonym = evaluate_results(Path(f'results_curated_1920_syn'),avg=avg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_curated_seed = pd.DataFrame.from_dict(results_curated_seed, orient='index', columns=['precision','recall','fscore','preds'])\n",
    "df_curated_synonym = pd.DataFrame.from_dict(results_curated_synonym, orient='index', columns=['precision','recall','fscore','preds'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\\begin{tabular}{lrr}\n\\toprule\n{} &  vertical &  horizontal \\\\\n\\midrule\nBERT\\_base sense centroid          &     0.691 &       0.536 \\\\\nBERT\\_base weighted sense centroid &     0.582 &       0.521 \\\\\nBERT\\_base perceptron              &     0.710 &       0.493 \\\\\nBERT\\_1900 sense centroid          &     0.700 &       0.554 \\\\\nBERT\\_1900 weighted sense centroid &     0.613 &       0.566 \\\\\nBERT\\_1900 perceptron              &     0.612 &       0.526 \\\\\nBERT\\_1850 sense centroid          &     0.658 &       0.563 \\\\\nBERT\\_1850 weighted sense centroid &     0.564 &       0.540 \\\\\nBERT\\_1850 perceptron              &     0.621 &       0.482 \\\\\n\\bottomrule\n\\end{tabular}\n\n"
     ]
    }
   ],
   "source": [
    "df_fscore = pd.concat([df_curated_seed['fscore'],df_curated_synonym['fscore']], axis=1)\n",
    "df_fscore = df_fscore.rename(index=renamed_columns)\n",
    "df_fscore.columns=['vertical','horizontal']\n",
    "print(df_fscore.to_latex())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Assess Statistical Significance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = evaluate_results(Path(\"results_1920/\"),avg=avg)\n",
    "\n",
    "selected = \"bert_centroid_sense_vector_bert_base_-1,-2,-3,-4_mean\"\n",
    "\n",
    "selected_pred = res[selected][3]\n",
    "print (selected, res[selected][:3], \"\\n\\nIs the difference significant?\\n\")\n",
    "\n",
    "for method,values in res.items():\n",
    "    if method != selected:\n",
    "        pred = values[3]\n",
    "        p_value = scipy.stats.ttest_rel(selected_pred[0],pred[0])[1]\n",
    "        if p_value<0.05:\n",
    "            print (method, values[:3], \"YES\")\n",
    "        else:\n",
    "            print (method, values[:3], \"NO p_value:\",round(p_value,5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python375jvsc74a57bd05d002d29fc59799ec58342307c50be960f7297ba8a27d201097020f600ad239f",
   "display_name": "Python 3.7.5 64-bit ('py37torch': conda)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  },
  "metadata": {
   "interpreter": {
    "hash": "5d002d29fc59799ec58342307c50be960f7297ba8a27d201097020f600ad239f"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}