iitkgp_scrapper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import json\n",
    "import time\n",
    "import sys\n",
    "import re\n",
    "import os\n",
    "from requests.exceptions import RequestException\n",
    "from bs4 import BeautifulSoup as bsoup\n",
    "from contextlib import closing\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.common.exceptions import WebDriverException"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def simple_get(url,sess):\n",
    "    try:\n",
    "        with closing(sess.get(url, stream=True)) as resp:\n",
    "            if is_good_response(resp):\n",
    "                return resp.content\n",
    "            else:\n",
    "                return None\n",
    "    except RequestException as e:\n",
    "        log_error('Error during requests to {0} : {1}'.format(url, str(e)))\n",
    "        return None\n",
    "\n",
    "def is_good_response(resp):\n",
    "    content_type = resp.headers['Content-Type'].lower()\n",
    "    return (resp.status_code == 200 \n",
    "            and content_type is not None \n",
    "            and content_type.find('html') > -1)\n",
    "\n",
    "def log_error(e):\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_file(data, filename, location='./college/'):\n",
    "    if not os.path.exists(location):\n",
    "        try:\n",
    "            os.makedirs(location)\n",
    "        except OSError as e:\n",
    "            if e.errno != errno.EEXIST:\n",
    "                raise\n",
    "    path=os.path.join(location, filename + \".\" + 'json')\n",
    "    with open(path,'w') as f:\n",
    "        json.dump(data, f, sort_keys=True, indent=4)\n",
    "    print('Done')\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def selenium_driver(url):\n",
    "    prof=[]\n",
    "    stud=[]\n",
    "    time.sleep(10)\n",
    "    options = webdriver.ChromeOptions()\n",
    "    options.add_argument('--ignore-certificate-errors')\n",
    "    options.add_argument('--incognito')\n",
    "    options.add_argument('--headless')\n",
    "    driver = webdriver.Chrome(\"/usr/bin/chromedriver\",options=options, service_log_path='./log/geckodriver.log')\n",
    "    driver.get(url)\n",
    "    ele=driver.find_elements(By.CSS_SELECTOR,\"button.btn.btn-primary.btn-xs\")\n",
    "    for x in range(len(ele)):\n",
    "        if ele[x].is_displayed():\n",
    "            driver.execute_script(\"arguments[0].click();\", ele[x])\n",
    "            time.sleep(1)\n",
    "    page_source = driver.page_source\n",
    "    soup = bsoup(page_source, 'html.parser')\n",
    "    table=soup.find(\"div\",{\"id\":\"showAlumniMembersDetails\"})\n",
    "    if table is not None:\n",
    "        table_body = table.find('tbody')\n",
    "        rows = table_body.find_all('tr')\n",
    "        for h in rows[0].find('td').find_all(re.compile(r'h\\d+')):\n",
    "            prof.append(h.text.strip())\n",
    "        rows.remove(rows[0])\n",
    "        flag=True\n",
    "        for row in rows:\n",
    "            tmp=[]\n",
    "            txt=row.find('td').text\n",
    "            if 'Ph.D.' in str(txt):\n",
    "                flag=True\n",
    "            elif 'MS' in str(txt):\n",
    "                flag=False\n",
    "            else:\n",
    "                if flag:\n",
    "                    #print('phd')\n",
    "                    tmp.append('phd')\n",
    "                    for a in str(row).split(\"<br/>\"):\n",
    "                        b=re.sub(r\"<[^>]*>\", \"\", a)\n",
    "                        tmp.append(b.strip())\n",
    "                        #print(b)\n",
    "                else:\n",
    "                    #print('ms')\n",
    "                    tmp.append('ms')\n",
    "                    for a in str(row).split(\"<br/>\"):\n",
    "                        b=re.sub(r\"<[^>]*>\", \"\", a)\n",
    "                        tmp.append(b.strip())\n",
    "                        #print(b)\n",
    "            if len(tmp) > 0:\n",
    "                stud.append(tmp)\n",
    "    else:\n",
    "        return None\n",
    "    driver.quit()\n",
    "    return (prof, stud)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "options = webdriver.FirefoxOptions()\n",
    "driver = webdriver.Firefox(options=options)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir(webdriver)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def iitkgp(url):\n",
    "    base_url=url\n",
    "    with requests.Session() as sess:\n",
    "        cont=simple_get('http://www.iitkgp.ac.in/department/CS',sess)\n",
    "        soup = bsoup(cont, 'html.parser')\n",
    "        link=soup.find('div',attrs={'class':'aboutTheDepartmentFacultyListing'}).find_all('a')\n",
    "        for l in link:\n",
    "            new_url=base_url+l['href'].split(';')[0]+\"#resp-tab4\"\n",
    "            yield new_url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#new_url='http://www.iitkgp.ac.in/department/CS/faculty/cs-abhij#resp-tab4'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "kgp={}\n",
    "file=1\n",
    "url='http://www.iitkgp.ac.in'\n",
    "for nurl in iitkgp(url):\n",
    "    try:\n",
    "        prof_stud=selenium_driver(nurl)\n",
    "        if prof_stud is not None:\n",
    "            kgp[\";\".join(prof_stud[0])]=prof_stud[1]\n",
    "    except WebDriverException as e:\n",
    "        write_file(kgp,'iitkgp'+str(file))\n",
    "        time.sleep(file*60)\n",
    "        file+=1\n",
    "        continue\n",
    "write_file(kgp,'iitkgp'+str(file))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "write_file(kgp,'iitkgp'+str(file))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'iitkgp'+str(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./college/iitkgp1.json') as json_file:\n",
    "    data = json.load(json_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "conn=0\n",
    "res=0\n",
    "for a in data:\n",
    "    res+=1+len(data[a])\n",
    "    conn += len(data[a])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for a in rows[0].find('td').find_all(re.compile(r'h\\d+')):\n",
    "    print(a.text.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1300"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(name1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "a=set(name1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1083"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import collections\n",
    "a=[item for item, count in collections.Counter(name1).items() if count > 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}