keywords_recall.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/numba/errors.py:137: UserWarning: Insufficiently recent colorama version found. Numba requires colorama >= 0.3.9\n",
      "  warnings.warn(msg)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from tqdm.notebook import tqdm\n",
    "import csv\n",
    "import pickle\n",
    "import time\n",
    "import swifter\n",
    "import multiprocessing as mp\n",
    "import torch\n",
    "from gensim import corpora, similarities, models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = '../../data_2020/'\n",
    "model_path = 'models/'\n",
    "other_path = 'others/'\n",
    "MIN_LEN = 8\n",
    "k = 10\n",
    "workers = 8\n",
    "thd=13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py:3051: DtypeWarning: Columns (2) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "train = pd.read_csv(data_path+'train_pre.csv')[['description_text_pre', 'paper_id']].fillna('')\n",
    "test = pd.read_csv(data_path+'test_pre.csv')[['description_text_pre']].fillna('')\n",
    "paper = pd.read_csv(data_path+'candidate_paper_pre.csv')[['paper_id', 'title_pro', 'keywords']].fillna('')\n",
    "paper2id={v:i for i,v in enumerate(paper['paper_id'].values)}\n",
    "id2paper={i:v for i,v in enumerate(paper['paper_id'].values)}\n",
    "train_ans=[paper2id[i] for i in train['paper_id'].values]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e8a1fc49695b4e35b0fcdaff5a85dbc1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=1205043.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "72b248ca096f4c71b91d885faffeb4ca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Dask Apply', max=96.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5615f779b18b439abbfd4cb6eef0637a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Dask Apply', max=96.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e2c88da22922499c8a8e050427a70364",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Dask Apply', max=96.0, style=ProgressStyle(description_wi…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "19959b82fd594b70a50c421ba0f6e84f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=62974.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "42a71efda3644721bf78483ce21679b9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=34428.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a83ad38ccb26415ea227b043830c916d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=838938.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "def df2idf_thd(docfreq, totaldocs, log_base=2.0, add=0.0):\n",
    "    idf = add + np.log(float(totaldocs) / docfreq) / np.log(log_base)\n",
    "    return idf if idf >= idf_thd else add\n",
    "def one(x):\n",
    "    return (x+1e-10)/(x+1e-10)\n",
    "tfidf =  models.TfidfModel.load(other_path+\"train_idf.model\")\n",
    "idf=set()\n",
    "for k,v in tqdm(tfidf.idfs.items()):\n",
    "    if v>thd:\n",
    "        idf.add(tfidf.id2word.id2token[k])\n",
    "def filter_len(string):\n",
    "    return set([word for word in string.split() if len(word) >= MIN_LEN])\n",
    "\n",
    "def top_k(string, k=3):\n",
    "    \n",
    "    ret = sorted(string.split(), key=lambda x: len(x), reverse=True)[:k]\n",
    "    ret = [kw for kw in ret if len(kw) >= MIN_LEN ]\n",
    "    return set(ret)\n",
    "\n",
    "train['description_text_pre'] = train['description_text_pre'].swifter.allow_dask_on_strings().apply(lambda x: filter_len(x))\n",
    "test['description_text_pre'] = test['description_text_pre'].swifter.allow_dask_on_strings().apply(lambda x: filter_len(x))\n",
    "paper['pseudo_keywords'] = paper['title_pro'].swifter.allow_dask_on_strings().apply(lambda x: top_k(x, k=k))\n",
    "train_pairs_500number_idf=pd.read_csv(data_path+'train_pairs_1000number_idf.csv',header=None)\n",
    "train = train.values\n",
    "test = test.values\n",
    "paper = paper.values\n",
    "\n",
    "for i in tqdm(range(train.shape[0])):\n",
    "    train[i][0]=train[i][0].intersection(idf)\n",
    "\n",
    "for i in tqdm(range(test.shape[0])):\n",
    "    test[i][0]=test[i][0].intersection(idf)    \n",
    "    \n",
    "for i in tqdm(range(paper.shape[0])):\n",
    "    paper[i][3]=paper[i][3].intersection(idf)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8577d51291a34b18b07a1725610ea0ee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "39641bdf45af44cb805f9dafd8dc6104",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a751c4e42cd44d9aa9e323596e4a096",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(<62974x838938 sparse matrix of type '<class 'numpy.float64'>'\n",
       " \twith 1817964 stored elements in List of Lists format>,\n",
       " <34428x838938 sparse matrix of type '<class 'numpy.float64'>'\n",
       " \twith 769219 stored elements in List of Lists format>)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from gensim import corpora\n",
    "from scipy.sparse import csr_matrix,lil_matrix\n",
    "train_keyword=[list(i[0]) for i in train]\n",
    "test_keyword=[list(i[0]) for i in test]\n",
    "paper_keyword=[list(i[3]) for i in paper]\n",
    "\n",
    "dictionary = corpora.Dictionary(train_keyword+test_keyword+paper_keyword)\n",
    "\n",
    "n_dict=len(dictionary.token2id)\n",
    "n_train=len(train)\n",
    "n_test=len(test)\n",
    "n_paper=len(paper)\n",
    "train_sparse=lil_matrix((n_train,n_dict))\n",
    "test_sparse=lil_matrix((n_test,n_dict))\n",
    "paper_sparse=lil_matrix((n_paper,n_dict))\n",
    "for i,vec in tqdm(enumerate(train_keyword)):\n",
    "    for e in vec:\n",
    "        train_sparse[i,dictionary.token2id[e]]=1\n",
    "        \n",
    "for i,vec in tqdm(enumerate(test_keyword)):\n",
    "    for e in vec:\n",
    "        test_sparse[i,dictionary.token2id[e]]=1\n",
    "    \n",
    "for i,vec in tqdm(enumerate(paper_keyword)):\n",
    "    for e in vec:\n",
    "        paper_sparse[i,dictionary.token2id[e]]=1 \n",
    "train_dot=train_sparse.dot(paper_sparse.T)\n",
    "train_dot=train_dot.tolil()\n",
    "\n",
    "test_dot=test_sparse.dot(paper_sparse.T)\n",
    "test_dot=test_dot.tolil()\n",
    "\n",
    "train_dot,test_dot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d553ec8291e44abe952d2028f661f4be",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=62974.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.06410582145012227, 163)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "hit=0\n",
    "overlap=0\n",
    "one=0\n",
    "for i in tqdm(range(len(train_ans))):\n",
    "    tfidf_30=[paper2id[pid] for pid in train_pairs_500number_idf.iloc[i,1:30].values]\n",
    "    if (train_ans[i] in train_dot.rows[i]):\n",
    "        if (train_ans[i] in tfidf_30):\n",
    "            overlap+=1\n",
    "        else:\n",
    "            hit+=1\n",
    "        if len(train_dot.rows[i])==1:\n",
    "            one+=1\n",
    "hit/len(train_ans),one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dids=train_pairs_500number_idf[0].values\n",
    "# with open(data_path+'train_pairs_500number_key2.csv', 'w', newline='') as f:\n",
    "#     w = csv.writer(f)\n",
    "#     for i in range(train_dot.rows.shape[0]):\n",
    "#         w.writerow([dids[i]]+[id2paper[idx] for idx in train_dot.rows[i]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_pairs_500number_s2v=pd.read_csv(data_path+'test_pairs_1000number_s2v.csv',header=None)\n",
    "dids=test_pairs_500number_s2v[0].values\n",
    "with open(data_path+'test_pairs_500number_key.csv', 'w', newline='') as f:\n",
    "    w = csv.writer(f)\n",
    "    for i in range(test_dot.rows.shape[0]):\n",
    "        w.writerow([dids[i]]+[id2paper[idx] for idx in test_dot.rows[i]])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}