train_dev2vec_models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from ast import literal_eval\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\morad\\Anaconda3\\lib\\site-packages\\tqdm\\std.py:648: FutureWarning: The Panel class is removed from pandas. Accessing it from the top-level namespace will also be removed in the next version\n",
      "  from pandas import Panel\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "from gensim.models import Doc2Vec\n",
    "from gensim.models.doc2vec import Doc2Vec\n",
    "from sklearn import utils\n",
    "from sklearn.model_selection import train_test_split\n",
    "from gensim.models.doc2vec import TaggedDocument"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## load datasets after cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "single_labels = pd.read_csv('..\\git_data_single_label.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev2vec_repos_dt = pd.read_csv(r'..\\dev2vec_repo_clean.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev2vec_repos_dt.text_all = dev2vec_repos_dt.text_all.apply(literal_eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dev2vec_repos_dt['text_all'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev2vec_issues_dt = pd.read_csv(r'..\\dev2vec_issue_clean.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev2vec_issues_dt.text_all = dev2vec_issues_dt.text_all.apply(literal_eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainset, testset = train_test_split(single_labels, test_size=0.1, random_state=45)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation, testset = train_test_split(testset, test_size=0.5, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainset_dev = list(trainset['gh_login'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "testset_dev = list(testset['gh_login'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_dev = list(validation['gh_login'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainset_dev2vec_repo= dev2vec_repos_dt[dev2vec_repos_dt['tags'].isin(trainset_dev)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "testset_dev2vec_repo= dev2vec_repos_dt[dev2vec_repos_dt['tags'].isin(testset_dev)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_dev2vec_repo= dev2vec_repos_dt[dev2vec_repos_dt['tags'].isin(validation_dev)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainset_dev2vec_issue= dev2vec_issues_dt[dev2vec_issues_dt['tags'].isin(trainset_dev)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "testset_dev2vec_issue= dev2vec_issues_dt[dev2vec_issues_dt['tags'].isin(testset_dev)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_dev2vec_issue= dev2vec_issues_dt[dev2vec_issues_dt['tags'].isin(validation_dev)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train dev2vec:repos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = trainset_dev2vec_repo.apply(\n",
    "    lambda r: TaggedDocument(words=r['text_all'], tags=[r.tags]), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = testset_dev2vec_repo.apply(\n",
    "    lambda r: TaggedDocument(words=r['text_all'], tags=[r.tags]), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "import multiprocessing\n",
    "cores = multiprocessing.cpu_count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Define the Cosine Similarity Function\n",
    "def cos_sim(av,bv):\n",
    "    return(sum(av*bv)/math.sqrt(sum(av*av)*sum(bv*bv)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vec_for_learning(model, tagged_docs):\n",
    "    sents = tagged_docs.values\n",
    "    model.random.seed(0)\n",
    "    targets, regressors = zip(*[(doc.tags[0], model.infer_vector(doc.words, epochs=25)) for doc in sents])\n",
    "    return targets, regressors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "dev2vec_repos_dmm.train(train.values, total_examples=len(train.values), epochs=dev2vec_repos_dmm.epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "dev2vec_repos_dmm = Doc2Vec(dm=1, vector_size=230, window=5, negative=5, min_count=5,epochs=15)\n",
    "dev2vec_repos_dmm.build_vocab(train.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dev2vec_repos_dmm.save(\"dev2vec_repos\", sep_limit=25000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 4min 14s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "y_train, X_train = vec_for_learning(dev2vec_repos_dmm, train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test, X_test = vec_for_learning(dev2vec_repos_dmm, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_train,labels=zip(*[(label,label) for label in trainset_dev2vec_repo['label']] )\n",
    "Y_test,labels=zip(*[(label,label) for label in testset_dev2vec_repo['label']] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "u_id_lst=[]\n",
    "for row in train.values:\n",
    "    u_id_lst.append(row.tags[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "sim_train=pd.DataFrame(columns=['u_id','cos_sim'], dtype=object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "for uid in range(0,len(u_id_lst)):\n",
    "    vec_train=X_train[uid]\n",
    "    username=u_id_lst[uid]\n",
    "    vec_user = dev2vec_repos_dmm.docvecs[username]\n",
    "    sim_train=sim_train.append({'u_id':username,'cos_sim':cos_sim(vec_train,vec_user)},ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9961270333075135"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(len(sim_train)-len(sim_train[sim_train['cos_sim']<0.85]))/len(sim_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Frontend         57\n",
       "Mobile           32\n",
       "Backend          19\n",
       "DataScientist    18\n",
       "DevOps           10\n",
       "Name: 0, dtype: int64"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(Y_test)[0].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "_________________________________________________________________________________________________________________\n",
    "\n",
    "X_train and X_test are the vector of developers in train and test sets. Y_train and Y_test are the associated labels. Vectors are the embedding reprsentation of the expertise of developers and can be fed into any classifiers or any other downstream task."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-------------------------------------------------------------------------------------------------------------\n",
    "## Train dev2vec:issues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = trainset_dev2vec_issue.apply(\n",
    "    lambda r: TaggedDocument(words=r['text_all'], tags=[r.tags]), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = testset_dev2vec_issue.apply(\n",
    "    lambda r: TaggedDocument(words=r['text_all'], tags=[r.tags]), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "dev2vec_issues_dmm = Doc2Vec(dm=1, vector_size=150, window=5, negative=5, min_count=5,epochs=25)\n",
    "dev2vec_issues_dmm.build_vocab(train.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "dev2vec_issues_dmm.train(train.values, total_examples=len(train.values), epochs=dev2vec_issues_dmm.epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev2vec_issues_dmm.save(\"dev2vec_issues\", sep_limit=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 27.9 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "y_train_issue, X_train_issue = vec_for_learning(dev2vec_issues_dmm, train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test_issue , X_test_issue = vec_for_learning(dev2vec_issues_dmm, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_train,labels=zip(*[(label,label) for label in dev2vec_issues_dmm['label']] )\n",
    "Y_test,labels=zip(*[(label,label) for label in dev2vec_issues_dmm['label']] )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "u_id_lst=[]\n",
    "for row in train.values:\n",
    "    u_id_lst.append(row.tags[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [],
   "source": [
    "sim_train=pd.DataFrame(columns=['u_id','cos_sim'], dtype=object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [],
   "source": [
    "for uid in range(0,len(u_id_lst)):\n",
    "    vec_train=X_train_issue[uid]\n",
    "    username=u_id_lst[uid]\n",
    "    vec_user = dev2vec_issues_dmm.docvecs[username]\n",
    "    sim_train=sim_train.append({'u_id':username,'cos_sim':cos_sim(vec_train,vec_user)},ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9668367346938775"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(len(sim_train)-len(sim_train[sim_train['cos_sim']<0.85]))/len(sim_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}