iitm_scrapper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup as bsoup\n",
    "import requests\n",
    "import sys\n",
    "from requests.exceptions import RequestException\n",
    "from contextlib import closing\n",
    "import time\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.common.exceptions import WebDriverException\n",
    "import re\n",
    "import os\n",
    "import json\n",
    "import time\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def simple_get(url,sess):\n",
    "    try:\n",
    "        with closing(sess.get(url, stream=True)) as resp:\n",
    "            if is_good_response(resp):\n",
    "                return resp.content\n",
    "            else:\n",
    "                return None\n",
    "    except RequestException as e:\n",
    "        log_error('Error during requests to {0} : {1}'.format(url, str(e)))\n",
    "        return None\n",
    "\n",
    "def is_good_response(resp):\n",
    "    content_type = resp.headers['Content-Type'].lower()\n",
    "    return (resp.status_code == 200 \n",
    "            and content_type is not None \n",
    "            and content_type.find('html') > -1)\n",
    "\n",
    "def log_error(e):\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def total_page(url, sess):\n",
    "    cont=simple_get(url, sess)\n",
    "    soup = bsoup(cont, 'html.parser')\n",
    "    homepage=soup.find(\"div\",{'id':'homepage'})\n",
    "    page=int(homepage.find(\"span\",{\"class\":\"paginate\"}).text.split()[-1])\n",
    "    return page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_extract(url, sess):\n",
    "    global iitm\n",
    "    deg = re.compile(\"PhD.*|M.Tech.*|MS|Postdoctoral|B.Tech|B.Tech(DD)\",re.IGNORECASE)\n",
    "    constraint=re.compile(\"Advisor|Next|Research|Guide|Faculty|Thesis\",re.IGNORECASE)\n",
    "    cont=simple_get(url, sess)\n",
    "    soup = bsoup(cont, 'html.parser')\n",
    "    homepage=soup.find(\"div\",{'id':'homepage'})\n",
    "    #page=int(homepage.find(\"span\",{\"class\":\"paginate\"}).text.split()[-1])\n",
    "    table=homepage.find('table')\n",
    "    rows=table.find_all('tr')\n",
    "    for row in rows:\n",
    "        td=row.find_all('td')\n",
    "        for i in td:\n",
    "            sch={}\n",
    "            i=str(i).split('<br/>')\n",
    "            for j in i:\n",
    "                b=re.sub(r\"<[^>]*>\", \"\", j).strip()\n",
    "                if b =='':\n",
    "                    continue\n",
    "                elif \":\" in b and constraint.match(b):\n",
    "                    tmp=b.split(\":\")\n",
    "                    sch[tmp[0]]=\":\".join(tmp[1:])\n",
    "                elif deg.match(b) is not None:\n",
    "                        sch['Degree']=b\n",
    "                else:\n",
    "                    if hasNumbers(b) or constraint.match(b):\n",
    "                        print(b)\n",
    "                        print('somthing wrong with name field')\n",
    "                    else:\n",
    "                        if sch.get('Name'): \n",
    "                            sch['Name']=sch.get('Name')+\"@\"+b\n",
    "                        else:\n",
    "                            sch['Name']=b\n",
    "            if len(sch) != 0:\n",
    "                iitm.append(sch)\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_file(data, filename, location='./college/'):\n",
    "    if not os.path.exists(location):\n",
    "        try:\n",
    "            os.makedirs(location)\n",
    "        except OSError as e:\n",
    "            if e.errno != errno.EEXIST:\n",
    "                raise\n",
    "    path=os.path.join(location, filename + \".\" + 'json')\n",
    "    with open(path,'w') as f:\n",
    "        json.dump(data, f, sort_keys=True, indent=4)\n",
    "    print('Done')\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hasNumbers(inputString):\n",
    "    return bool(re.search(r'\\d', inputString))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Anirban was a part-time PhD scholar (from IBM Research). He discontinued from the program as on Jan 2018.\n",
      "somthing wrong with name field\n",
      "Anirban was a part-time PhD scholar (from IBM Research). He discontinued from the program as on Jan 2018.\n",
      "somthing wrong with name field\n",
      "Done\n",
      "CPU times: user 1.96 s, sys: 20.4 ms, total: 1.98 s\n",
      "Wall time: 5min 38s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#mtech=\"U3R1ZGVudHMhMTMkMSQk\"\n",
    "#btech_dd=\"U3R1ZGVudHMhMTQkMSQk\"\n",
    "#btech=\"U3R1ZGVudHMhMTUkMSQk\"\n",
    "filename='iitm'\n",
    "args=[\"U3R1ZGVudHMhMTUkMSQk\",\"U3R1ZGVudHMhMTQkMSQk\",\"U3R1ZGVudHMhMTMkMSQk\"]\n",
    "iitm=[]\n",
    "url='http://www.cse.iitm.ac.in/listpeople.php?arg=MyQxJCQ=&%20page=1&ipp=50'\n",
    "tmp_url='http://www.cse.iitm.ac.in/listpeople.php?arg=MyQxJCQ=&%20page=1&ipp=50'\n",
    "with requests.Session() as sess:\n",
    "    for arg in range(len(args)+1):\n",
    "        page=total_page(url, sess)\n",
    "        for i in range(2, page+2):\n",
    "            data_extract(url, sess)\n",
    "            url=url.split('page')[0]+\"page=\"+str(i)+\"&ipp=50\"\n",
    "            time.sleep(random.randint(10,15))\n",
    "        if arg < len(args):\n",
    "            tmp_url1=tmp_url.split('MyQxJCQ')\n",
    "            url=tmp_url1[0]+args[arg]+tmp_url1[1]\n",
    "write_file(iitm, filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Anirban was a part-time PhD scholar (from IBM Research). He discontinued from the program as on Jan 2018.\n",
      "somthing wrong with name field\n"
     ]
    }
   ],
   "source": [
    "iitm=[]\n",
    "#args=[\"U3R1ZGVudHMhMTUkMSQk\",\"U3R1ZGVudHMhMTQkMSQk\",\"U3R1ZGVudHMhMTMkMSQk\"]\n",
    "args=[]\n",
    "url='http://www.cse.iitm.ac.in/listpeople.php?arg=MyQxJCQ=&%20page=1&ipp=50'\n",
    "#tmp_url='http://www.cse.iitm.ac.in/listpeople.php?arg=MyQxJCQ=&%20page=1&ipp=50'\n",
    "with requests.Session() as sess:\n",
    "    for arg in range(len(args)+1):\n",
    "        #page=total_page(url, sess)\n",
    "        for i in range(1, 2):\n",
    "            data_extract(url, sess)\n",
    "            url=url.split('page')[0]+\"page=\"+str(i)+\"&ipp=50\"\n",
    "            time.sleep(random.randint(10,15))\n",
    "        if arg < len(args):\n",
    "            tmp_url1=tmp_url.split('MyQxJCQ')\n",
    "            url=tmp_url1[0]+args[arg]+tmp_url1[1]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1300"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(iitm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "write_file(iitm,'iitm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "3\n",
      "4\n",
      "5\n"
     ]
    }
   ],
   "source": [
    "for i in range(2,6):\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "250"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(iitm1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scrapely import Scraper\n",
    "s = Scraper()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "a={u'Name': 'Ramesh .S',\n",
    " u'Degree': 'MS Scholar,  Jul 1999 -  Jan 2002',\n",
    " u'Advisor(s)': 'Sukhendu Das',\n",
    " u'Thesis Title': ' Edginess Image for Face Recognition'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}