web_search.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from googlesearch import search \n",
    "import import_ipynb\n",
    "from bs4 import BeautifulSoup as bsoup\n",
    "from multiprocessing.dummy import Pool\n",
    "import requests\n",
    "import sys\n",
    "from requests.exceptions import RequestException\n",
    "from contextlib import closing\n",
    "import re\n",
    "import json\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def content_Preprocess(content):\n",
    "    if content is not None:\n",
    "        title=''\n",
    "        text=''\n",
    "        soup = bsoup(content, 'html.parser')\n",
    "        try:\n",
    "            title += soup.find('title').getText()\n",
    "        except AttributeError as error:\n",
    "            print(error)\n",
    "        for script in soup([\"script\", \"style\"]):\n",
    "            script.extract()    # rip it out\n",
    "        # get text\n",
    "        text += soup.get_text()\n",
    "        # break into lines and remove leading and trailing space on each\n",
    "        lines = (line.strip() for line in text.splitlines())\n",
    "        # break multi-headlines into a line each\n",
    "        chunks = (phrase.strip() for line in lines for phrase in line.split(\"  \"))\n",
    "        # drop blank lines\n",
    "        text = '\\n'.join(chunk for chunk in chunks if chunk)\n",
    "        return (title, text)\n",
    "    else:\n",
    "        return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def simple_get(url):\n",
    "    try:\n",
    "        with closing(requests.get(url, stream=True)) as resp:\n",
    "            if is_good_response(resp):\n",
    "                return resp.content\n",
    "            else:\n",
    "                return None\n",
    "    except RequestException as e:\n",
    "        log_error('Error during requests to {0} : {1}'.format(url, str(e)))\n",
    "        return None\n",
    "\n",
    "def is_good_response(resp):\n",
    "    content_type = resp.headers['Content-Type'].lower()\n",
    "    return (resp.status_code == 200 \n",
    "            and content_type is not None \n",
    "            and content_type.find('html') > -1)\n",
    "\n",
    "\n",
    "def log_error(e):\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_json(filename, directory='./tmp/'):\n",
    "    path=os.path.join(directory, filename + \".\" + 'json')\n",
    "    with open(path) as f:\n",
    "        data = json.load(f)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_colleage(name):\n",
    "    ent_name=name.replace(' +',\"_\")\n",
    "    content=simple_get('http://en.wikipedia.org/wiki/'+ent_name)\n",
    "    if content is not None:\n",
    "        soup = bsoup(content, 'html.parser')\n",
    "        table = soup.find('table', attrs={'class':'infobox biography vcard'})\n",
    "        if table is not None:\n",
    "            table_body = table.find('tbody')\n",
    "            rows = table_body.find_all('tr')\n",
    "            for row in rows:\n",
    "                col_hd = row.find('th')\n",
    "                col=row.find('td')\n",
    "                if col_hd is not None and col is not None:\n",
    "                    col = col.text.strip()\n",
    "                    col_hd=col_hd.text.strip()\n",
    "                    if \"Alma\" in col_hd:\n",
    "                        return (name,col.split(\"\\n\")[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "def multiple_process(ent_name, get_colleage, proc=2):\n",
    "    p=Pool(proc)\n",
    "    try:\n",
    "        cllg=p.map(get_colleage, ent_name)\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "    return cllg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ent_name=[]\n",
    "pid_name_entid=load_json('pid2ent2','./')\n",
    "for a in pid_name_entid:\n",
    "    ent_name.append(a[1])\n",
    "a=multiple_process(ent_name, get_colleage)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"^(?=.*?\\b[A|a]lan\\b)(?=.*?\\b[T|t]uring\\b)(?=.*?\\b[D|d]octorate\\b).*$\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "query = 'Alan turing AND doctorate institute'\n",
    "repeat=[]\n",
    "for url in search(query, stop=20):\n",
    "    if 'YouTube' not in url:\n",
    "        if not any((a in url or url in a) for a in repeat):\n",
    "            r=simple_get(url)\n",
    "            if content_Preprocess(r) is not None:\n",
    "                title_text=content_Preprocess(r)\n",
    "                print(url)\n",
    "                match=re.search(\"([Aa]lan|[Tt]uring).*([Dd]octorate).*?\\.\",title_text[1])\n",
    "                if match is not None:\n",
    "                    print(match.group())\n",
    "                repeat.append(url)\n",
    "            else:\n",
    "                continue\n",
    "        else:\n",
    "            continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}