Python_LSA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda2\\lib\\site-packages\\gensim\\utils.py:860: UserWarning: detected Windows; aliasing chunkize to chunkize_serial\n",
      "  warnings.warn(\"detected Windows; aliasing chunkize to chunkize_serial\")\n"
     ]
    }
   ],
   "source": [
    "from gensim import corpora, models, similarities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "documents = [\"Human machine interface for lab computer applications\",\n",
    "             \"A survey of user opinion of computer system response time\",\n",
    "             \"The EPS user interface management system\",\n",
    "             \"System and human system engineering testing of EPS\",\n",
    "             \"Relation of user perceived response time to error measurement\",\n",
    "             \"The generation of random binary unordered trees\",\n",
    "             \"The intersection graph of paths in trees\",\n",
    "             \"Graph minors IV Widths of trees and well quasi ordering\",\n",
    "             \"Graph minors A survey\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Human machine interface for lab computer applications', 'A survey of user opinion of computer system response time', 'The EPS user interface management system', 'System and human system engineering testing of EPS', 'Relation of user perceived response time to error measurement', 'The generation of random binary unordered trees', 'The intersection graph of paths in trees', 'Graph minors IV Widths of trees and well quasi ordering', 'Graph minors A survey']\n"
     ]
    }
   ],
   "source": [
    "print(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# remove common words and tokenize them\n",
    "stoplist = set('for a of the and to in'.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "texts = [[word for word in document.lower().split() if word not in stoplist] for document in documents]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['human', 'machine', 'interface', 'lab', 'computer', 'applications'], ['survey', 'user', 'opinion', 'computer', 'system', 'response', 'time'], ['eps', 'user', 'interface', 'management', 'system'], ['system', 'human', 'system', 'engineering', 'testing', 'eps'], ['relation', 'user', 'perceived', 'response', 'time', 'error', 'measurement'], ['generation', 'random', 'binary', 'unordered', 'trees'], ['intersection', 'graph', 'paths', 'trees'], ['graph', 'minors', 'iv', 'widths', 'trees', 'well', 'quasi', 'ordering'], ['graph', 'minors', 'survey']]\n"
     ]
    }
   ],
   "source": [
    "print(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['human', 'machine', 'interface', 'lab', 'computer', 'applications', 'survey', 'user', 'opinion', 'computer', 'system', 'response', 'time', 'eps', 'user', 'interface', 'management', 'system', 'system', 'human', 'system', 'engineering', 'testing', 'eps', 'relation', 'user', 'perceived', 'response', 'time', 'error', 'measurement', 'generation', 'random', 'binary', 'unordered', 'trees', 'intersection', 'graph', 'paths', 'trees', 'graph', 'minors', 'iv', 'widths', 'trees', 'well', 'quasi', 'ordering', 'graph', 'minors', 'survey']\n"
     ]
    }
   ],
   "source": [
    "# remove words those appear only once\n",
    "all_tokens = sum(texts, [])\n",
    "\n",
    "print(all_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "set(['generation', 'random', 'iv', 'engineering', 'relation', 'measurement', 'unordered', 'binary', 'management', 'ordering', 'machine', 'quasi', 'testing', 'paths', 'lab', 'applications', 'intersection', 'perceived', 'widths', 'well', 'error', 'opinion'])\n"
     ]
    }
   ],
   "source": [
    "tokens_once = set(word for word in set(all_tokens) if all_tokens.count(word) ==1)\n",
    "\n",
    "print(tokens_once)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['human', 'interface', 'computer'], ['survey', 'user', 'computer', 'system', 'response', 'time'], ['eps', 'user', 'interface', 'system'], ['system', 'human', 'system', 'eps'], ['user', 'response', 'time'], ['trees'], ['graph', 'trees'], ['graph', 'minors', 'trees'], ['graph', 'minors', 'survey']]\n"
     ]
    }
   ],
   "source": [
    "texts = [[word for word in text if word not in tokens_once]\n",
    "         for text in texts]\n",
    "\n",
    "print(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dictionary(12 unique tokens: [u'minors', u'graph', u'system', u'trees', u'eps']...)\n"
     ]
    }
   ],
   "source": [
    "dictionary = corpora.Dictionary(texts)\n",
    "\n",
    "print(dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dictionary.save('deerwester.dict')  # save as binary file at the dictionary at local directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dictionary.save_as_text('deerwester_text.dict')  # save as text file at the local directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{u'minors': 11, u'graph': 10, u'system': 6, u'trees': 9, u'eps': 8, u'computer': 1, u'survey': 5, u'user': 7, u'human': 2, u'time': 4, u'interface': 0, u'response': 3}\n"
     ]
    }
   ],
   "source": [
    "print(dictionary.token2id) # show pairs of \"word : word-ID number\" "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(1, 1), (2, 1)]\n"
     ]
    }
   ],
   "source": [
    "new_doc = \"Human computer interaction\" # temporary data to see role of below function\n",
    "\n",
    "new_vec = dictionary.doc2bow(new_doc.lower().split()) # return \"word-ID : Frequency of appearance\"\"\n",
    "print(new_vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[(0, 1), (1, 1), (2, 1)], [(1, 1), (3, 1), (4, 1), (5, 1), (6, 1), (7, 1)], [(0, 1), (6, 1), (7, 1), (8, 1)], [(2, 1), (6, 2), (8, 1)], [(3, 1), (4, 1), (7, 1)], [(9, 1)], [(9, 1), (10, 1)], [(9, 1), (10, 1), (11, 1)], [(5, 1), (10, 1), (11, 1)]]\n"
     ]
    }
   ],
   "source": [
    "corpus = [dictionary.doc2bow(text) for text in texts]\n",
    "\n",
    "print(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "corpora.MmCorpus.serialize('deerwester.mm', corpus) # save corpus at local directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[(0, 1.0), (1, 1.0), (2, 1.0)], [(1, 1.0), (3, 1.0), (4, 1.0), (5, 1.0), (6, 1.0), (7, 1.0)], [(0, 1.0), (6, 1.0), (7, 1.0), (8, 1.0)], [(2, 1.0), (6, 2.0), (8, 1.0)], [(3, 1.0), (4, 1.0), (7, 1.0)], [(9, 1.0)], [(9, 1.0), (10, 1.0)], [(9, 1.0), (10, 1.0), (11, 1.0)], [(5, 1.0), (10, 1.0), (11, 1.0)]]\n"
     ]
    }
   ],
   "source": [
    "corpus = corpora.MmCorpus('deerwester.mm') # try to load the saved corpus from local\n",
    "\n",
    "print(list(corpus)) # to show corpus which was read above, need to print(list( )) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dictionary(12 unique tokens: [u'minors', u'graph', u'system', u'trees', u'eps']...)\n"
     ]
    }
   ],
   "source": [
    "dictionary = corpora.Dictionary.load('deerwester.dict') # try to load saved dic.from local\n",
    "\n",
    "print(dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MmCorpus(9 documents, 12 features, 28 non-zero entries)\n"
     ]
    }
   ],
   "source": [
    "print(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TfidfModel(num_docs=9, num_nnz=28)\n"
     ]
    }
   ],
   "source": [
    "tfidf = models.TfidfModel(corpus) # step 1 -- initialize a model\n",
    "\n",
    "print(tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<gensim.interfaces.TransformedCorpus object at 0x0000000009BA2CF8>\n"
     ]
    }
   ],
   "source": [
    "corpus_tfidf = tfidf[corpus]  # map corpus object into tfidf space\n",
    "\n",
    "print(corpus_tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, 0.5773502691896257), (1, 0.5773502691896257), (2, 0.5773502691896257)]\n",
      "[(1, 0.44424552527467476), (3, 0.44424552527467476), (4, 0.44424552527467476), (5, 0.44424552527467476), (6, 0.3244870206138555), (7, 0.3244870206138555)]\n",
      "[(0, 0.5710059809418182), (6, 0.4170757362022777), (7, 0.4170757362022777), (8, 0.5710059809418182)]\n",
      "[(2, 0.49182558987264147), (6, 0.7184811607083769), (8, 0.49182558987264147)]\n",
      "[(3, 0.6282580468670046), (4, 0.6282580468670046), (7, 0.45889394536615247)]\n",
      "[(9, 1.0)]\n",
      "[(9, 0.7071067811865475), (10, 0.7071067811865475)]\n",
      "[(9, 0.5080429008916749), (10, 0.5080429008916749), (11, 0.695546419520037)]\n",
      "[(5, 0.6282580468670046), (10, 0.45889394536615247), (11, 0.6282580468670046)]\n"
     ]
    }
   ],
   "source": [
    "for doc in corpus_tfidf: # show tfidf-space mapped words\n",
    "    print(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LsiModel(num_terms=12, num_topics=2, decay=1.0, chunksize=20000)\n"
     ]
    }
   ],
   "source": [
    "lsi = models.LsiModel(corpus_tfidf, id2word=dictionary, num_topics=2) # initialize LSI \n",
    "print(lsi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<gensim.interfaces.TransformedCorpus object at 0x0000000009BA2D30>\n"
     ]
    }
   ],
   "source": [
    "corpus_lsi = lsi[corpus_tfidf] # create a double wrapper over the original corpus\n",
    "print(corpus_lsi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "topic = lsi.print_topics(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, u'0.703*\"trees\" + 0.538*\"graph\" + 0.402*\"minors\" + 0.187*\"survey\" + 0.061*\"system\" + 0.060*\"response\" + 0.060*\"time\" + 0.058*\"user\" + 0.049*\"computer\" + 0.035*\"interface\"'), (1, u'-0.460*\"system\" + -0.373*\"user\" + -0.332*\"eps\" + -0.328*\"interface\" + -0.320*\"time\" + -0.320*\"response\" + -0.293*\"computer\" + -0.280*\"human\" + -0.171*\"survey\" + 0.161*\"trees\"')]\n"
     ]
    }
   ],
   "source": [
    "print(topic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, 0.066007833960903692), (1, -0.52007033063618491)]\n",
      "[(0, 0.19667592859142474), (1, -0.76095631677000508)]\n",
      "[(0, 0.089926399724464118), (1, -0.7241860626752511)]\n",
      "[(0, 0.075858476521781251), (1, -0.63205515860034311)]\n",
      "[(0, 0.10150299184980116), (1, -0.57373084830029575)]\n",
      "[(0, 0.70321089393783121), (1, 0.16115180214025779)]\n",
      "[(0, 0.87747876731198327), (1, 0.16758906864659406)]\n",
      "[(0, 0.90986246868185805), (1, 0.14086553628719031)]\n",
      "[(0, 0.61658253505692828), (1, -0.053929075663893447)]\n"
     ]
    }
   ],
   "source": [
    "for doc in corpus_lsi:\n",
    "    print(doc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lsi.save('model.lsi')  # save output model at local directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LsiModel(num_terms=12, num_topics=2, decay=1.0, chunksize=20000)\n"
     ]
    }
   ],
   "source": [
    "lsi = models.LsiModel.load('model.lsi') # try to load above saved model\n",
    "print(lsi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(1, 1), (2, 1)]\n"
     ]
    }
   ],
   "source": [
    "doc = \"Human computer interaction\"  # give new document to calculate similarity degree with already obtained topics\n",
    "\n",
    "vec_bow = dictionary.doc2bow(doc.lower().split())  # put newly obtained document to existing dictionary object\n",
    "print(vec_bow)  # show result of above "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, 0.07910475117444854), (1, -0.57328352430794027)]\n"
     ]
    }
   ],
   "source": [
    "vec_lsi = lsi[vec_bow] # convert new document (henceforth, call it \"query\") to LSI space\n",
    "print(vec_lsi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MatrixSimilarity<9 docs, 2 features>\n"
     ]
    }
   ],
   "source": [
    "index = similarities.MatrixSimilarity(lsi[corpus]) # transform corpus to LSI space and indexize it\n",
    "print(index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "index.save('deerwester.index') # save index object at local directory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "index = similarities.MatrixSimilarity.load('deerwester.index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MatrixSimilarity<9 docs, 2 features>\n"
     ]
    }
   ],
   "source": [
    "print(index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0.99994081  0.99467081  0.99994278  0.999879    0.99935204 -0.08804217\n",
      " -0.0515742  -0.02366471  0.1938726 ]\n"
     ]
    }
   ],
   "source": [
    "sims = index[vec_lsi] # calculate degree of similarity of the query to existing corpus\n",
    "print(sims)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, 0.99994081), (1, 0.99467081), (2, 0.99994278), (3, 0.999879), (4, 0.99935204), (5, -0.08804217), (6, -0.0515742), (7, -0.023664713), (8, 0.1938726)]\n"
     ]
    }
   ],
   "source": [
    "print(list(enumerate(sims))) # output (document_number , document similarity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(2, 0.99994278), (0, 0.99994081), (3, 0.999879), (4, 0.99935204), (1, 0.99467081), (8, 0.1938726), (7, -0.023664713), (6, -0.0515742), (5, -0.08804217)]\n"
     ]
    }
   ],
   "source": [
    "sims = sorted(enumerate(sims), key=lambda item: -item[1])  # sort output object as per similarity ( largest similarity document comes first )\n",
    "print(sims)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}