LSI-Cosine.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Work\\Anaconda\\envs\\py35\\lib\\site-packages\\gensim\\utils.py:855: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n"
     ]
    }
   ],
   "source": [
    "# imports:\n",
    "import time;\n",
    "import urllib.request\n",
    "import csv\n",
    "import os\n",
    "import itertools\n",
    "import logging\n",
    "import json\n",
    "import numpy as np\n",
    "import gensim\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib.parse import urlencode\n",
    "from gensim.utils import smart_open, simple_preprocess\n",
    "from gensim.corpora.wikicorpus import _extract_pages, filter_wiki\n",
    "from gensim.parsing.preprocessing import STOPWORDS\n",
    "from xml.etree.cElementTree import iterparse\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "logging.basicConfig(format='%(levelname)s : %(message)s', level=logging.INFO)\n",
    "logging.root.level = logging.INFO\n",
    "model_name=\"LSI-Cosine1\"\n",
    "id2word_wiki=gensim.corpora.Dictionary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking folder : ArtGeoHeaHisLitMatMusNatRelTec\n"
     ]
    }
   ],
   "source": [
    "# List Categories:\n",
    "#category_list = [\"Mathematics\",\"Technology\",\"Music\"]\n",
    "category_list = [\"Mathematics\",\"Technology\",\"Music\",\"History\",\"Geography\",\"Arts\",\"Health\",\"Nature\",\"Religion\",\"Literature\"]\n",
    "#category_list = [\"1977 introductions\", \"Language\" ,\"Arts\", \"Asia\", \"Asthma\", \"Automobiles\", \"BRICS nation\", \"Belief\", \"Climate\", \"Crime\", \"Culture\", \"Dance\", \"Deserts\", \"Dolls\", \"Earth\", \"Engines\", \"Fishing\", \"Folklore\", \"Games\", \"Geography\", \"Glass\", \"Government agencies\", \"Health\", \"History\", \"Humans\", \"Hygiene\", \"India\", \"Industry\", \"Internet\", \"Law\", \"Life\", \"Literature\", \"Mathematics\", \"Matter\", \"Millionaires\", \"Music\", \"Nature\", \"Nothing\", \"Parties\", \"Peace\", \"Politics\", \"Religion\", \"Sexology\", \"Society\", \"Songs\", \"Space\", \"Technology\", \"Television\", \"Transport\", \"Water sports\"]\n",
    "test_folder =  \"./test/\"\n",
    "root_folder = \"./Simplex/\"\n",
    "list.sort(category_list)\n",
    "folder_name=''.join([x[0]+x[1]+x[2] for x in category_list])\n",
    "root_folder='./'+folder_name+'/'\n",
    "\n",
    "print(\"Checking folder : \"+folder_name)\n",
    "if not os.path.exists(root_folder):\n",
    "    os.mkdir(root_folder)\n",
    "    \n",
    "if not os.path.exists(root_folder+model_name):\n",
    "    os.mkdir(root_folder+model_name)\n",
    "\n",
    "    \n",
    "\n",
    "wiki_bow_path = root_folder+'wiki_bow.mm'\n",
    "wiki_dict_path = root_folder+'wiki_dict.dict'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scanning CSV\n"
     ]
    }
   ],
   "source": [
    "# Download Page Ids:\n",
    "#https://petscan.wmflabs.org/?language=en&project=wikipedia&depth=1&format=csv&categories=mathematics&doit=Do it!\n",
    "print (\"Scanning CSV\")\n",
    "for cat in category_list:\n",
    "    if not os.path.exists(root_folder+cat+\".csv\"):\n",
    "        url=\"https://petscan.wmflabs.org/?language=en&project=wikipedia&depth=1&format=csv&doit=Do%20it!&categories=\"+cat\n",
    "        urllib.request.urlretrieve(url, root_folder+cat+\".csv\")\n",
    "        print(\"Downloading \",cat+\".csv\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done downloading\n"
     ]
    }
   ],
   "source": [
    "# CSV to XML Download data:\n",
    "\n",
    "def get_data(ids,output_file):\n",
    "    url=\"https://en.wikipedia.org/w/api.php?action=query&prop=revisions&rvprop=content&format=xml&pageids=\"+ids\n",
    "    req = urllib.request.urlopen(url)\n",
    "    if req.getcode() == 200:\n",
    "        soup = BeautifulSoup(req.read(), 'html.parser')\n",
    "        s = soup.find_all('page')\n",
    "        for si in s:\n",
    "            output_file.write(str(si))\n",
    "\n",
    "def batch_train(file):\n",
    "    output_file = open(file.replace(\".csv\",\".xml\"), 'a', encoding=\"utf8\")\n",
    "    output_file.write(\"<pages>\")\n",
    "                    \n",
    "    csvReader = csv.reader(open(file,'r'))\n",
    "    totalRecords = sum(1 for row in csv.reader(open(file,'r',encoding=\"UTF-8\")) )\n",
    "    print (file.replace(\".csv\",\".xml\"),totalRecords)\n",
    "    start = 0\n",
    "    end = start + 50\n",
    "   \n",
    "    while (start <= totalRecords):\n",
    "        pageIds = \"\"\n",
    "        for row in itertools.islice(csv.reader(open(file,'r',encoding=\"UTF-8\")),start,end):\n",
    "            pageIds = pageIds + row[2] + \"|\"\n",
    "        \n",
    "        get_data(pageIds,output_file)\n",
    "        start = end + 1\n",
    "        end = start + 50\n",
    "        if end> totalRecords:\n",
    "            end=totalRecords\n",
    "            \n",
    "    output_file.write(\"</pages>\")\n",
    "    \n",
    "\n",
    "for path, subdirs, files in os.walk(root_folder):\n",
    "    for file in files:\n",
    "        if file.endswith('.csv'):\n",
    "            if not os.path.exists(root_folder+file.replace(\".csv\",\".xml\")):\n",
    "                print(\"Downloading \",root_folder+file.replace(\".csv\",\".xml\"))\n",
    "                batch_train(path + file)\n",
    "\n",
    "print (\"Done downloading\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Train Model\n",
    "def head(stream, n=10):\n",
    "    \"\"\"Convenience fnc: return the first `n` elements of the stream, as plain list.\"\"\"\n",
    "    return list(itertools.islice(stream, n))\n",
    "\n",
    "def my_extract_pages(f):\n",
    "    elems = (elem for _, elem in iterparse(f, events=(\"end\",)))\n",
    "    page_tag = \"rev\"\n",
    "    for elem in elems:\n",
    "        if elem.tag == page_tag and elem.text != None:\n",
    "            text = elem.text\n",
    "            yield text\n",
    "            elem.clear()\n",
    "\n",
    "def tokenize(text):\n",
    "    return [token for token in simple_preprocess(text) if token not in STOPWORDS]\n",
    "\n",
    "def iter_wiki(dump_file):\n",
    "    \"\"\"Yield each article from the Wikipedia dump, as a `(title, tokens)` 2-tuple.\"\"\"\n",
    "    ignore_namespaces = 'Wikipedia Category File Portal Template MediaWiki User Help Book Draft'.split()\n",
    "    for text in my_extract_pages(smart_open(dump_file)):\n",
    "        text = filter_wiki(text)\n",
    "        tokens = tokenize(text)\n",
    "        if len(tokens) < 50:\n",
    "            continue  # ignore short articles and various meta-articles\n",
    "        yield tokens\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO : loading Dictionary object from ./ArtGeoHeaHisLitMatMusNatRelTec/wiki_dict.dict\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Model\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO : loaded ./ArtGeoHeaHisLitMatMusNatRelTec/wiki_dict.dict\n",
      "INFO : loaded corpus index from ./ArtGeoHeaHisLitMatMusNatRelTec/wiki_bow.mm.index\n",
      "INFO : initializing corpus reader from ./ArtGeoHeaHisLitMatMusNatRelTec/wiki_bow.mm\n",
      "INFO : accepted corpus with 1968 documents, 37814 features, 419342 non-zero entries\n",
      "INFO : collecting document frequencies\n",
      "INFO : PROGRESS: processing document #0\n",
      "INFO : calculating IDF weights for 1968 documents and 37813 features (419342 matrix non-zeros)\n",
      "INFO : using serial LSI version on this node\n",
      "INFO : updating model with new documents\n",
      "INFO : preparing a new chunk of documents\n",
      "INFO : using 100 extra samples and 2 power iterations\n",
      "INFO : 1st phase: constructing (37814, 110) action matrix\n",
      "INFO : orthonormalizing (37814, 110) action matrix\n",
      "INFO : 2nd phase: running dense svd on (110, 1968) matrix\n",
      "INFO : computing the final decomposition\n",
      "INFO : keeping 10 factors (discarding 74.597% of energy spectrum)\n",
      "INFO : processed documents up to #1968\n",
      "INFO : topic #0(5.167): -0.288*\"mobile\" + -0.121*\"engineering\" + -0.115*\"software\" + -0.106*\"digital\" + -0.105*\"user\" + -0.096*\"users\" + -0.095*\"internet\" + -0.087*\"web\" + -0.085*\"learning\" + -0.084*\"technologies\"\n",
      "INFO : topic #1(3.487): 0.576*\"mobile\" + -0.260*\"engineering\" + 0.143*\"phone\" + -0.114*\"technological\" + -0.110*\"energy\" + 0.109*\"wireless\" + 0.108*\"phones\" + 0.096*\"location\" + 0.094*\"sms\" + -0.087*\"industrial\"\n",
      "INFO : topic #2(3.039): 0.167*\"technical\" + 0.155*\"document\" + 0.155*\"content\" + 0.151*\"documentation\" + 0.141*\"learning\" + -0.139*\"energy\" + 0.132*\"xml\" + 0.129*\"web\" + -0.125*\"signal\" + 0.122*\"software\"\n",
      "INFO : topic #3(2.879): -0.243*\"mobile\" + -0.198*\"engineering\" + -0.165*\"technological\" + 0.154*\"iso\" + 0.141*\"document\" + 0.140*\"xml\" + -0.122*\"innovation\" + 0.121*\"format\" + 0.119*\"documentation\" + 0.113*\"message\"\n",
      "INFO : topic #4(2.718): -0.592*\"engineering\" + 0.219*\"digital\" + -0.168*\"mobile\" + -0.146*\"iso\" + -0.137*\"engineers\" + 0.118*\"internet\" + 0.118*\"video\" + -0.113*\"standards\" + -0.101*\"technical\" + 0.093*\"learning\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "print(\"Training Model\")\n",
    "class WikiCorpus(object):\n",
    "    def __init__(self, dump_file, dictionary, clip_docs=None):\n",
    "        \"\"\"\n",
    "        Parse the first `clip_docs` Wikipedia documents from file `dump_file`.\n",
    "        Yield each document in turn, as a list of tokens (unicode strings).\n",
    "        \n",
    "        \"\"\"\n",
    "        self.dump_file = dump_file\n",
    "        self.dictionary = dictionary\n",
    "        self.clip_docs = clip_docs\n",
    "    \n",
    "    def __iter__(self):\n",
    "        for tokens in itertools.islice(iter_wiki(self.dump_file), self.clip_docs):\n",
    "            yield self.dictionary.doc2bow(tokens)\n",
    "    \n",
    "    def __len__(self):\n",
    "        return self.clip_docs\n",
    "\n",
    "if not (os.path.exists(wiki_bow_path) and  os.path.exists(wiki_dict_path)):\n",
    "    for path, subdirs, files in os.walk(root_folder):\n",
    "        del subdirs[:]\n",
    "        for file in files:\n",
    "            if file.endswith('.xml'):\n",
    "                doc_path = path + file\n",
    "                stream = iter_wiki(doc_path)\n",
    "                doc_stream = (tokens for tokens in iter_wiki(doc_path))\n",
    "                id2word_wiki.merge_with(gensim.corpora.Dictionary(doc_stream))\n",
    "\n",
    "    id2word_wiki.filter_extremes(no_below=10, no_above=0.1)\n",
    "    \n",
    "    # create a stream of bag-of-words vectors\n",
    "    wiki_corpus = WikiCorpus(doc_path, id2word_wiki)\n",
    "    \n",
    "    id2word_wiki.save(wiki_dict_path) \n",
    "    gensim.corpora.MmCorpus.serialize(wiki_bow_path, wiki_corpus)\n",
    "\n",
    "id2word_wiki = gensim.corpora.Dictionary.load(wiki_dict_path)\n",
    "mm_corpus = gensim.corpora.MmCorpus(wiki_bow_path)\n",
    "clipped_corpus = gensim.utils.ClippedCorpus(mm_corpus, 4000) \n",
    "tfidf_model = gensim.models.TfidfModel(mm_corpus, id2word=id2word_wiki)\n",
    "lsi_model = lsi_model = gensim.models.LsiModel(tfidf_model[mm_corpus], id2word=id2word_wiki, num_topics=len(category_list))\n",
    "print(\"done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./ArtGeoHeaHisLitMatMusNatRelTec/Arts.xml\n",
      "10\n",
      "1389\n",
      "./ArtGeoHeaHisLitMatMusNatRelTec/Geography.xml\n",
      "10\n",
      "978\n",
      "./ArtGeoHeaHisLitMatMusNatRelTec/Health.xml\n",
      "10\n",
      "2214\n",
      "./ArtGeoHeaHisLitMatMusNatRelTec/History.xml\n",
      "10\n",
      "1468\n",
      "./ArtGeoHeaHisLitMatMusNatRelTec/Literature.xml\n",
      "10\n",
      "1776\n",
      "./ArtGeoHeaHisLitMatMusNatRelTec/Mathematics.xml\n",
      "10\n",
      "1057\n",
      "./ArtGeoHeaHisLitMatMusNatRelTec/Music.xml\n",
      "10\n",
      "1349\n",
      "./ArtGeoHeaHisLitMatMusNatRelTec/Nature.xml\n",
      "10\n",
      "481\n",
      "./ArtGeoHeaHisLitMatMusNatRelTec/Religion.xml\n",
      "10\n",
      "845\n",
      "./ArtGeoHeaHisLitMatMusNatRelTec/Technology.xml\n",
      "10\n",
      "1968\n",
      "Calcuating centroid\n"
     ]
    }
   ],
   "source": [
    "def calculate_centroid(topic_docs):\n",
    "    print(topic_docs)\n",
    "    test_doc = [tokens for tokens in iter_wiki(topic_docs)]\n",
    "    part = [lsi_model[id2word_wiki.doc2bow(tokens)] for tokens in test_doc]\n",
    "    \n",
    "    topic_dic={}\n",
    "    print(len(category_list))\n",
    "    for i in range(len(category_list)):\n",
    "        topic_dic[i]=0\n",
    "        \n",
    "    for doc in part:\n",
    "        for p in doc:\n",
    "            topic_dic[p[0]] += p[1]\n",
    "    print(len(part))\n",
    "    centroid = [(x, topic_dic[x]/len(part)) for x in range(len(category_list))]\n",
    "    return centroid\n",
    "    \n",
    "centroids_dict={}\n",
    "for path, subdirs, files in os.walk(root_folder):\n",
    "    del subdirs[:]\n",
    "    for file in files:\n",
    "        if file.endswith('.xml'):\n",
    "            doc_path = path + file\n",
    "            centroid = calculate_centroid(doc_path)\n",
    "            centroids_dict[file.replace(\".xml\",\"\")]=centroid\n",
    "\n",
    "print(\"Calcuating centroid\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def drawgraph(x_label,y,file,text_data,topic):\n",
    "    x = np.arange(len(x_label))  # the x locations for the groups\n",
    "    width = 0.1       # the width of the bars\n",
    "\n",
    "    fig, ax = plt.subplots()\n",
    "    fig.set_figheight(6)\n",
    "    fig.set_figwidth(8)\n",
    "    rects = ax.bar(x, y, width, color='blue')\n",
    "    ax.set_ylabel('Probabilities')\n",
    "    ax.set_xlabel('Categories')\n",
    "    ax.set_title('Topic Distribution of: ' + topic)\n",
    "    ax.title.set_position([.5, 1.2])\n",
    "    ax.set_ylim([0,1])\n",
    "    \n",
    "    def autolabel(rects):\n",
    "        \"\"\"\n",
    "        Attach a text label above each bar displaying its height\n",
    "        \"\"\"\n",
    "        for rect in rects:\n",
    "            height = rect.get_height()\n",
    "            ax.text(rect.get_x() - rect.get_width()*2, 1.05 * height,round(height,2))\n",
    "    \n",
    "\n",
    "    autolabel(rects)\n",
    "    plt.tight_layout(pad=6)\n",
    "    plt.xticks(x,category_list,rotation=90)\n",
    "    \n",
    "    plt.savefig(file.replace(\".xml\",\"_\")+str(len(category_list))+'.png')\n",
    "    plt.close('all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing results\n",
      " Done \n"
     ]
    }
   ],
   "source": [
    "# Test \n",
    "def get_part(testFile):\n",
    "    test_doc = [tokens for tokens in iter_wiki(testFile)]\n",
    "    part = [lsi_model[id2word_wiki.doc2bow(tokens)] for tokens in test_doc]\n",
    "    return part\n",
    "\n",
    "results=\"\"\n",
    "print(\"Testing results\")\n",
    "for path, subdirs, files in os.walk(test_folder):\n",
    "    for file in files:\n",
    "        if file.endswith('.xml'):\n",
    "            doc_path = test_folder + file\n",
    "            path=get_part(doc_path)\n",
    "            results+=file.replace(\".xml\",\"\")+\"\\n\\n\"\n",
    "            graph_data=[]\n",
    "            text_data=\"\"\n",
    "            graph_topic=[]\n",
    "            for topic in category_list:\n",
    "                \n",
    "                cos_dis=np.mean([gensim.matutils.cossim(p1, p2) for p1, p2 in zip([centroids_dict[topic]], path)])\n",
    "                text_data +=  topic+\":\"+str(cos_dis)+\"\\n\"\n",
    "                if(np.mean([gensim.matutils.cossim(p1, p2) for p1, p2 in zip([centroids_dict[topic]], path)])>0):\n",
    "                    graph_data.append(np.mean([gensim.matutils.cossim(p1, p2) for p1, p2 in zip([centroids_dict[topic]], path)]))\n",
    "                    graph_topic.append(topic)\n",
    "            results+=text_data+\"\\n\\n\\n\\n\"\n",
    "                \n",
    "            #drawgraph(list(centroids_dict.keys()),graph_data,root_folder+model_name+'/'+file,text_data,file)\n",
    "            drawgraph(graph_topic,graph_data,root_folder+model_name+'/'+file,text_data,file)\n",
    "output_file = open(root_folder+model_name+\"/\"+model_name+str(len(category_list))+\".txt\", 'w', encoding=\"utf8\")\n",
    "output_file.write(results)\n",
    "output_file.close()\n",
    "print(\" Done \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:py35]",
   "language": "python",
   "name": "conda-env-py35-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}