sent2vec_embedding.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### imports\n",
    "***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/numba/errors.py:137: UserWarning: Insufficiently recent colorama version found. Numba requires colorama >= 0.3.9\n",
      "  warnings.warn(msg)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch.nn import functional as F\n",
    "\n",
    "from fse.models.base_s2v import BaseSentence2VecModel\n",
    "import sent2vec\n",
    "from nltk import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from string import punctuation\n",
    "import fasttext\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import swifter\n",
    "from tqdm.notebook import tqdm\n",
    "import csv\n",
    "import pickle\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### load data\n",
    "***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = '../../data_2020/'\n",
    "model_path = 'BioWordVec_PubMed_MIMICIII_d200.bin'\n",
    "other_path = 'others/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# candidate_paper = pd.read_csv(data_path+\"candidate_paper_pre.csv\")\n",
    "train_data = pd.read_csv(data_path+\"train_pre.csv\")\n",
    "valid_data = pd.read_csv(data_path+\"test_pre.csv\")\n",
    "# candidate_paper = candidate_paper[~candidate_paper['paper_id'].isnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = train_data.fillna('none')\n",
    "valid_data = valid_data.fillna('none')\n",
    "# candidate_paper = candidate_paper.fillna('none')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>description_id</th>\n",
       "      <th>paper_id</th>\n",
       "      <th>description_text</th>\n",
       "      <th>key_text</th>\n",
       "      <th>key_text_pre</th>\n",
       "      <th>description_text_pre</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>77bef2</td>\n",
       "      <td>5c0f7919da562944ac759a0f</td>\n",
       "      <td>Angiogenesis is reflected as newly formed vess...</td>\n",
       "      <td>Moreover, Wnt-1-inducible secreted protein-1 (...</td>\n",
       "      <td>moreover wnt-1-inducible wnt 1 inducible secre...</td>\n",
       "      <td>angiogenesi reflect newly form vessel endothel...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>42360e</td>\n",
       "      <td>5c1360beda56295a0896fda3</td>\n",
       "      <td>Cardiac fibrosis is a common process in remode...</td>\n",
       "      <td>There is evidence showing that the down-regula...</td>\n",
       "      <td>there evidence show down-regulation down regul...</td>\n",
       "      <td>cardiac fibrosi common process remodel heart M...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9bf5e0</td>\n",
       "      <td>5d1b36e83a55ac0a0e8bb84e</td>\n",
       "      <td>Agmatine, formed by the decarboxylation of L-a...</td>\n",
       "      <td>Agmatine, formed by the decarboxylation of L-a...</td>\n",
       "      <td>agmatine form decarboxylation l-arginine l arg...</td>\n",
       "      <td>agmatine form decarboxylation l-arginine l arg...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>22e485</td>\n",
       "      <td>5d2709fd3a55ac2cfc28108f</td>\n",
       "      <td>The ob gene product leptin has been demonstrat...</td>\n",
       "      <td>The aminoguanidine carboxylate, BVT.12777 (Fig...</td>\n",
       "      <td>the aminoguanidine carboxylate BVT.12777 figur...</td>\n",
       "      <td>the ob gene product leptin demonstrate activat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>30856c</td>\n",
       "      <td>55a392d1c91b587b095b6fcc</td>\n",
       "      <td>Lauterbach M et al., have concluded at the end...</td>\n",
       "      <td>Lauterbach M et , have concluded at the end of...</td>\n",
       "      <td>lauterbach M et conclude end study germany ana...</td>\n",
       "      <td>lauterbach M et al. conclude end study germany...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  description_id                  paper_id  \\\n",
       "0         77bef2  5c0f7919da562944ac759a0f   \n",
       "1         42360e  5c1360beda56295a0896fda3   \n",
       "2         9bf5e0  5d1b36e83a55ac0a0e8bb84e   \n",
       "3         22e485  5d2709fd3a55ac2cfc28108f   \n",
       "4         30856c  55a392d1c91b587b095b6fcc   \n",
       "\n",
       "                                    description_text  \\\n",
       "0  Angiogenesis is reflected as newly formed vess...   \n",
       "1  Cardiac fibrosis is a common process in remode...   \n",
       "2  Agmatine, formed by the decarboxylation of L-a...   \n",
       "3  The ob gene product leptin has been demonstrat...   \n",
       "4  Lauterbach M et al., have concluded at the end...   \n",
       "\n",
       "                                            key_text  \\\n",
       "0  Moreover, Wnt-1-inducible secreted protein-1 (...   \n",
       "1  There is evidence showing that the down-regula...   \n",
       "2  Agmatine, formed by the decarboxylation of L-a...   \n",
       "3  The aminoguanidine carboxylate, BVT.12777 (Fig...   \n",
       "4  Lauterbach M et , have concluded at the end of...   \n",
       "\n",
       "                                        key_text_pre  \\\n",
       "0  moreover wnt-1-inducible wnt 1 inducible secre...   \n",
       "1  there evidence show down-regulation down regul...   \n",
       "2  agmatine form decarboxylation l-arginine l arg...   \n",
       "3  the aminoguanidine carboxylate BVT.12777 figur...   \n",
       "4  lauterbach M et conclude end study germany ana...   \n",
       "\n",
       "                                description_text_pre  \n",
       "0  angiogenesi reflect newly form vessel endothel...  \n",
       "1  cardiac fibrosi common process remodel heart M...  \n",
       "2  agmatine form decarboxylation l-arginine l arg...  \n",
       "3  the ob gene product leptin demonstrate activat...  \n",
       "4  lauterbach M et al. conclude end study germany...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>description_id</th>\n",
       "      <th>description_text</th>\n",
       "      <th>key_text</th>\n",
       "      <th>key_text_pre</th>\n",
       "      <th>description_text_pre</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00032c</td>\n",
       "      <td>Refer to Table 2 or Methods for a brief descri...</td>\n",
       "      <td>Colons (:) indicated interaction terms..</td>\n",
       "      <td>colon indicate interaction terms..</td>\n",
       "      <td>refer table method brief description variable ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>000676</td>\n",
       "      <td>Sixty-nine female subjects with a mean age of ...</td>\n",
       "      <td>Handedness was evaluated according to the proc...</td>\n",
       "      <td>handedness evaluate accord procedure propose a...</td>\n",
       "      <td>sixty-nine sixty nine female subject mean age ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>000b24</td>\n",
       "      <td>Our behavioral and imaging findings differed f...</td>\n",
       "      <td>Recently, Chiu et   used a modified IGT, namel...</td>\n",
       "      <td>recently chiu et use modified IGT namely sooch...</td>\n",
       "      <td>our behavioral imaging finding differ previou ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>000c20</td>\n",
       "      <td>A novel Ehrlichia transmitted by Amblyomma ame...</td>\n",
       "      <td>ruminantium, caused transient febrile illness,...</td>\n",
       "      <td>ruminantium cause transient febrile illness fo...</td>\n",
       "      <td>A novel ehrlichium transmit amblyomma american...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>000c90</td>\n",
       "      <td>The dorsal fronto-striatal circuit plays an im...</td>\n",
       "      <td>One of these functions is set-shifting, which ...</td>\n",
       "      <td>one function set-shifting set shifting refer a...</td>\n",
       "      <td>the dorsal fronto-striatal fronto striatal cir...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  description_id                                   description_text  \\\n",
       "0         00032c  Refer to Table 2 or Methods for a brief descri...   \n",
       "1         000676  Sixty-nine female subjects with a mean age of ...   \n",
       "2         000b24  Our behavioral and imaging findings differed f...   \n",
       "3         000c20  A novel Ehrlichia transmitted by Amblyomma ame...   \n",
       "4         000c90  The dorsal fronto-striatal circuit plays an im...   \n",
       "\n",
       "                                            key_text  \\\n",
       "0          Colons (:) indicated interaction terms..    \n",
       "1  Handedness was evaluated according to the proc...   \n",
       "2  Recently, Chiu et   used a modified IGT, namel...   \n",
       "3  ruminantium, caused transient febrile illness,...   \n",
       "4  One of these functions is set-shifting, which ...   \n",
       "\n",
       "                                        key_text_pre  \\\n",
       "0                 colon indicate interaction terms..   \n",
       "1  handedness evaluate accord procedure propose a...   \n",
       "2  recently chiu et use modified IGT namely sooch...   \n",
       "3  ruminantium cause transient febrile illness fo...   \n",
       "4  one function set-shifting set shifting refer a...   \n",
       "\n",
       "                                description_text_pre  \n",
       "0  refer table method brief description variable ...  \n",
       "1  sixty-nine sixty nine female subject mean age ...  \n",
       "2  our behavioral imaging finding differ previou ...  \n",
       "3  A novel ehrlichium transmit amblyomma american...  \n",
       "4  the dorsal fronto-striatal fronto striatal cir...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### preprocess\n",
    "***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c97a5f13d9e2493da625fdfc1554ed25",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Dask Apply', max=96.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00c9ef244954457a8e4c9324752bd885",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Dask Apply', max=96.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "28a71d9a46a144b7b9a245b20cb75baa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Dask Apply', max=96.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ddb087eaf914f1fb76e599d733fbbe6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Dask Apply', max=96.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "stop_words = set(stopwords.words('english'))\n",
    "\n",
    "def preprocess_sentence(text):\n",
    "    text = text.replace('/', ' / ')\n",
    "    text = text.replace('.-', ' .- ')\n",
    "    text = text.replace('.', ' . ')\n",
    "    text = text.replace('\\'', ' \\' ')\n",
    "    text = text.lower()\n",
    "    tokens = [token for token in word_tokenize(text) if token not in punctuation and token not in stop_words]\n",
    "    return ' '.join(tokens)\n",
    "\n",
    "train_data['key_text_pre'] = train_data['key_text_pre'].swifter.allow_dask_on_strings().apply(lambda x: preprocess_sentence(x)).values\n",
    "train_data['description_text_pre'] = train_data['description_text_pre'].swifter.allow_dask_on_strings().apply(lambda x: preprocess_sentence(x)).values\n",
    "valid_data['key_text_pre'] = valid_data['key_text_pre'].swifter.allow_dask_on_strings().apply(lambda x: preprocess_sentence(x)).values\n",
    "valid_data['description_text_pre'] = valid_data['description_text_pre'].swifter.allow_dask_on_strings().apply(lambda x: preprocess_sentence(x)).values\n",
    "# candidate_paper['title_pro'] = candidate_paper['title_pro'].swifter.allow_dask_on_strings().apply(lambda x: preprocess_sentence(x)).values\n",
    "# candidate_paper['abstract_pre'] = candidate_paper['abstract_pre'].swifter.allow_dask_on_strings().apply(lambda x: preprocess_sentence(x)).values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### embedding\n",
    "***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "model = fasttext.load_model(model_path)\n",
    "\n",
    "def get_vec_ft(x):\n",
    "    vec = [model[word] for word in x.split()]\n",
    "    if len(vec) == 0:\n",
    "        return np.nan\n",
    "    else:\n",
    "        return np.mean(np.array(vec), axis=0)\n",
    "\n",
    "def random_vec():\n",
    "    return np.random.normal(0, 0.1, 200)\n",
    "\n",
    "paper2embedding = {}\n",
    "description2embedding = {}\n",
    "\n",
    "# for i, r in tqdm(candidate_paper.iterrows(), total=candidate_paper.shape[0]):\n",
    "#     paper2embedding[r['paper_id']] = {}\n",
    "#     # abstract\n",
    "#     ab = r['abstract_pre']\n",
    "#     if ab == 'no_content' or ab == 'none' or ab == 'n o n e':\n",
    "#         paper2embedding[r['paper_id']]['abstract'] = random_vec()\n",
    "#     else:\n",
    "#         paper2embedding[r['paper_id']]['abstract'] = get_vec_ft(ab)\n",
    "#     # title\n",
    "#     title = r['title_pro']\n",
    "#     paper2embedding[r['paper_id']]['title'] = get_vec_ft(title)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(other_path+'paper2embedding_s2v.pkl', 'wb') as f:\n",
    "#     pickle.dump(paper2embedding, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "becbc5ea5eb9414c9d29774ea6ef42d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=62974.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "170d313f4e4448f2903c0175903191a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=34428.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for i, r in tqdm(train_data.iterrows(), total=train_data.shape[0]):\n",
    "    description2embedding[r['description_id']+'_train'] = {}\n",
    "    # description_text\n",
    "    dcp = r['description_text_pre']\n",
    "    if dcp == 'none' or dcp == 'n o n e':\n",
    "        description2embedding[r['description_id']+'_train']['description_text'] = random_vec()\n",
    "    else:\n",
    "        description2embedding[r['description_id']+'_train']['description_text'] = get_vec_ft(dcp)\n",
    "    # key_text\n",
    "    key = r['key_text_pre']\n",
    "    if key == 'none' or key == 'n o n e':\n",
    "        description2embedding[r['description_id']+'_train']['key_text'] = random_vec()\n",
    "    else:\n",
    "        description2embedding[r['description_id']+'_train']['key_text'] = get_vec_ft(key)\n",
    "        \n",
    "for i, r in tqdm(valid_data.iterrows(), total=valid_data.shape[0]):\n",
    "    description2embedding[r['description_id']+'_test'] = {}\n",
    "    # description_text\n",
    "    dcp = r['description_text_pre']\n",
    "    if dcp == 'none' or dcp == 'n o n e':\n",
    "        description2embedding[r['description_id']+'_test']['description_text'] = random_vec()\n",
    "    else:\n",
    "        description2embedding[r['description_id']+'_test']['description_text'] = get_vec_ft(dcp)\n",
    "    # key_text\n",
    "    key = r['key_text_pre']\n",
    "    if key == 'none' or key == 'n o n e':\n",
    "        description2embedding[r['description_id']+'_test']['key_text'] = random_vec()\n",
    "    else:\n",
    "        description2embedding[r['description_id']+'_test']['key_text'] = get_vec_ft(key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(other_path+'description2embedding_s2v.pkl', 'wb') as f:\n",
    "    pickle.dump(description2embedding, f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}