reading_writing_wikidata_folder.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import import_ipynb\n",
    "import pandas as pd\n",
    "import requests\n",
    "from multiprocessing.dummy import Pool\n",
    "import re\n",
    "from SPARQLWrapper import SPARQLWrapper, JSON\n",
    "from collections import defaultdict\n",
    "from knowledge_extract import *\n",
    "from os import listdir\n",
    "from os.path import isfile, join\n",
    "import traceback\n",
    "from datetime import date\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%lsmagic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "people=pd.read_csv('./neurotree/people_cs.tsv', sep='\\t', encoding='utf-8',  escapechar='\\\\')\n",
    "connect=pd.read_csv('./neurotree/connect_cs.tsv', sep='\\t', encoding='latin-1',  escapechar='\\\\')\n",
    "people['name']=people['firstname'].fillna('')+\" \"+people['middlename'].fillna('')+\" \"+people['lastname'].fillna('')\n",
    "#people['name']=people['name'].str.replace('.','',regex=False)\n",
    "people['name']=people['name'].str.replace(' +',' ',regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "people.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "people[people['name'].str.contains('Anup Kumar')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_id(args):\n",
    "    ent=get_entity_id(args[1],\"https://www.wikidata.org/w/api.php\")\n",
    "    return (args[0], args[1], ent)\n",
    "result=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'./tmp14-09-2019_00:00:00/'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'./tmp'+date.today().strftime(\"%d-%m-%Y_%H:%M:%S\")+'/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'11-09-2019 09:54:20'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datetime.datetime.now().strftime(\"%d-%m-%Y %H:%M:%S\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "names=people[['pid','name']]\n",
    "names = [tuple(x) for x in names.values]\n",
    "p=Pool(3)\n",
    "for a in p.imap_unordered(get_id, names):\n",
    "        result.append(a)\n",
    "mod_result=[t for t in result if t[2] is not None]\n",
    "with open('pid2ent2.json', 'w') as fp:\n",
    "    json.dump(mod_result, fp, sort_keys=True, indent=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(mod_result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "b=[]\n",
    "for a in mod_result:\n",
    "    b.append(a[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(set(b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_inst(entity):\n",
    "    global directory\n",
    "    props=['P27','P106','P101','P184','P185','P1026','P802','P1066','P800','P856','P2456','P2381','P864',\n",
    "       'P108','P69','P512','P166','P463','P39','P496']\n",
    "    try:\n",
    "        inst=Knowledge_extract(entity[2], props)\n",
    "        if not inst.flag:\n",
    "            inst.write_json(directory=directory)\n",
    "    except Exception as e:\n",
    "        print(entity[2])\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_json(filename, directory='./tmp/'):\n",
    "    path=os.path.join(directory, filename + \".\" + 'json')\n",
    "    with open(path) as f:\n",
    "        data = json.load(f)\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_filesInDir(directory):\n",
    "    jsonfiles = [f for f in listdir(directory) if isfile(join(directory, f))]\n",
    "    entity=[f.split('.')[0] for f in jsonfiles]\n",
    "    return entity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Q15637844\n",
      "CPU times: user 12min 56s, sys: 1min 5s, total: 14min 2s\n",
      "Wall time: 2h 44min 15s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "directory='./tmp'+date.today().strftime(\"%d_%m_%Y\")+'/'\n",
    "Knowledge_extract.get_entities(directory=directory)\n",
    "res=load_json('pid2ent2','./')\n",
    "p=Pool(3)\n",
    "try:\n",
    "    p.map(create_inst,res)\n",
    "except Exception as e:\n",
    "    traceback.print_exc()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for a in mod_result:\n",
    "    if a[1]==\"Anup Kumar\":\n",
    "        print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a=[]\n",
    "for r in mod_result:\n",
    "    a.append(r[2])\n",
    "b=set(a)\n",
    "dup=[]\n",
    "for e in b:\n",
    "    count=a.count(e)\n",
    "    if count > 1:\n",
    "        dup.append(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for d in dup:\n",
    "    for r in mod_result:\n",
    "        if r[2]==d:\n",
    "            print(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mod_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}