git_repos_data_collection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import requests \n",
    "from github import Github\n",
    "from github import RateLimitExceededException\n",
    "from github import UnknownObjectException\n",
    "from datetime import datetime, timedelta\n",
    "import pickle\n",
    "import time\n",
    "import itertools\n",
    "import os\n",
    "import calendar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_roles = pd.read_csv('..\\dev_name_users.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collect Repository dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GitHubScrapper:\n",
    "\n",
    "    def __init__(self):\n",
    "        auth_git = #put your github authentication key here\n",
    "        self.g = Github(auth_git)\n",
    "\n",
    "    def user_info_extracter(self, users):\n",
    "\n",
    "        #users_iter = iter(users)\n",
    "\n",
    "        for u in users:\n",
    "            try:\n",
    "                user = self.g.get_user(u)\n",
    "                try:\n",
    "                    name = user.name\n",
    "\n",
    "                except:\n",
    "                    name = \"none\"\n",
    "                    \n",
    "                try:\n",
    "                    u_id = user.login\n",
    "\n",
    "                except:\n",
    "                    u_id = \"none\"\n",
    "\n",
    "                try:\n",
    "                    date = user.created_at\n",
    "                except:\n",
    "                    date = \"none\"\n",
    "\n",
    "\n",
    "                try:\n",
    "                    activity = user.last_modified\n",
    "                except:\n",
    "                    activity = \"none\"\n",
    "\n",
    "                try:\n",
    "                    html_url = user.html_url\n",
    "                except:\n",
    "                    html_url = \"none\"\n",
    "\n",
    "                try:\n",
    "                    email = user.email\n",
    "                except:\n",
    "                    email = \"none\"\n",
    "\n",
    "                try:\n",
    "                    location = user.location\n",
    "                except:\n",
    "                    location = \"none\"\n",
    "\n",
    "                try:\n",
    "                    blog = user.blog\n",
    "                except:\n",
    "                    blog = \"none\"\n",
    "\n",
    "                try:\n",
    "                    bio = user.bio\n",
    "                except:\n",
    "                    bio = \"none\"\n",
    "\n",
    "                try:\n",
    "                    reps = user.get_repos()\n",
    "                except:\n",
    "                    reps = \"none\"\n",
    "\n",
    "                full_rep = {}\n",
    "                cnt = 0\n",
    "                for repository in reps:\n",
    "                    repo = {}\n",
    "                    try:\n",
    "                        repo_name = repository.name\n",
    "                    except:\n",
    "                        repo_name = \"none\"\n",
    "\n",
    "                    repo['repo_name'] = repo_name\n",
    "\n",
    "                    try:\n",
    "                        readme = repository.get_contents('README.md').decoded_content\n",
    "                    except:\n",
    "                        readme = \"none\"\n",
    "\n",
    "                    repo['readme'] = readme\n",
    "                    \n",
    "                    try:\n",
    "                        repo_topic = repository.topics\n",
    "                    except:\n",
    "                        repo_topic = \"none\"\n",
    "\n",
    "                    repo['topics'] = repo_topic\n",
    "                    \n",
    "                    \n",
    "                    try:\n",
    "                        repo_about = repository.description\n",
    "                    except:\n",
    "                        repo_about = \"none\"\n",
    "\n",
    "                    repo['about'] = repo_about\n",
    "\n",
    "                    try:\n",
    "                        stars = repository.stargazers_count\n",
    "                    except:\n",
    "                        stars = \"none\"\n",
    "\n",
    "                    repo['stars'] = stars\n",
    "\n",
    "                    try:\n",
    "                        perc = 0\n",
    "                        lang_perc = {}\n",
    "                        for key, value in repository.get_languages().items():\n",
    "                            perc += value\n",
    "\n",
    "                        for key, value in repository.get_languages().items():\n",
    "                            percentage = value / perc * 100\n",
    "                            lang_perc[key] = percentage\n",
    "                        repo['languages'] = lang_perc\n",
    "                    except:\n",
    "                        repo['languages'] = \"none\"\n",
    "\n",
    "\n",
    "                    try:\n",
    "                        total_commits = repository.get_commits().totalCount\n",
    "                    except:\n",
    "                        total_commits =1\n",
    "            \n",
    "                    try:\n",
    "                        total_commits_author = repository.get_commits(author=user.login).totalCount\n",
    "                    except:\n",
    "                        total_commits_author=0\n",
    "                        \n",
    "                    try:    \n",
    "                        author_contribution = total_commits_author / total_commits * 100\n",
    "                    except:\n",
    "                        author_contribution=0\n",
    "                        \n",
    "                    try:   \n",
    "                        commit_files = [item.filename.split(\".\")[-1] for cm in repository.get_commits(author=user.login) for item in cm.files]\n",
    "                    except:\n",
    "                        commit_files = []\n",
    "\n",
    "                    try:\n",
    "                        extension_count = {}\n",
    "                        for extension in commit_files:\n",
    "                            if extension not in extension_count.keys():\n",
    "                                extension_count[extension] = 1\n",
    "                            else:\n",
    "                                extension_count[extension]+=1\n",
    "\n",
    "                    except:\n",
    "                        extension_count = {}\n",
    "\n",
    "\n",
    "                    repo['total_commits'] = total_commits\n",
    "                    repo['author_commits'] = total_commits_author\n",
    "                    repo['author_contribution'] = author_contribution\n",
    "\n",
    "                    repo['files'] = extension_count\n",
    "                    full_rep[cnt] = repo\n",
    "                    cnt += 1\n",
    "\n",
    "\n",
    "                data = {}\n",
    "\n",
    "                data['name'] = name\n",
    "                data['u_id'] = u_id\n",
    "                data['date'] = date\n",
    "                data['activity'] = activity\n",
    "                data['url'] = html_url\n",
    "                data['email'] = email\n",
    "                data['location'] = location\n",
    "                data['blog'] = blog\n",
    "                data['bio'] = bio\n",
    "                data['repository'] = full_rep\n",
    "\n",
    "                    \n",
    "        \n",
    "                pickle.dump(data, open(('../' + u_id + '.pickle'), 'wb'))\n",
    "            \n",
    "\n",
    "                print(u_id, \"-------------------> done\")\n",
    "\n",
    "            except RateLimitExceededException:\n",
    "                now = datetime.now()\n",
    "                current_time = now.strftime(\"%H:%M:%S\")\n",
    "                search_rate_limit = self.g.get_rate_limit().search\n",
    "                print('current time:{}'.format(current_time))\n",
    "                #time.sleep(sleep_time)\n",
    "                time.sleep(3600)\n",
    "                continue\n",
    "            except UnknownObjectException:\n",
    "                continue\n",
    "                \n",
    "    def get_user_details(self):\n",
    "        self.user_info_extracter(user_roles)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pronix -------------------> done\n"
     ]
    }
   ],
   "source": [
    "gg = GitHubScrapper()\n",
    "gg.get_user_details()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}