documentRetrievalPractice.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## In this we'll understand the difference in performance of raw count vs tf-idf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import graphlab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "people = graphlab.SFrame('people_wiki.gl/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\"><table frame=\"box\" rules=\"cols\">\n",
       "    <tr>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">URI</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">name</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">text</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">&lt;http://dbpedia.org/resou<br>rce/Digby_Morrell&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Digby Morrell</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">digby morrell born 10<br>october 1979 is a former ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">&lt;http://dbpedia.org/resou<br>rce/Alfred_J._Lewy&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Alfred J. Lewy</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">alfred j lewy aka sandy<br>lewy graduated from ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">&lt;http://dbpedia.org/resou<br>rce/Harpdog_Brown&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Harpdog Brown</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">harpdog brown is a singer<br>and harmonica player who ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">&lt;http://dbpedia.org/resou<br>rce/Franz_Rottensteiner&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Franz Rottensteiner</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">franz rottensteiner born<br>in waidmannsfeld lower ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">&lt;http://dbpedia.org/resou<br>rce/G-Enka&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">G-Enka</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">henry krvits born 30<br>december 1974 in tallinn ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">&lt;http://dbpedia.org/resou<br>rce/Sam_Henderson&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Sam Henderson</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">sam henderson born<br>october 18 1969 is an ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">&lt;http://dbpedia.org/resou<br>rce/Aaron_LaCrate&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Aaron LaCrate</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">aaron lacrate is an<br>american music producer ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">&lt;http://dbpedia.org/resou<br>rce/Trevor_Ferguson&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Trevor Ferguson</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">trevor ferguson aka john<br>farrow born 11 november ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">&lt;http://dbpedia.org/resou<br>rce/Grant_Nelson&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Grant Nelson</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">grant nelson born 27<br>april 1971 in london  ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">&lt;http://dbpedia.org/resou<br>rce/Cathy_Caruth&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Cathy Caruth</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">cathy caruth born 1955 is<br>frank h t rhodes ...</td>\n",
       "    </tr>\n",
       "</table>\n",
       "[10 rows x 3 columns]<br/>\n",
       "</div>"
      ],
      "text/plain": [
       "Columns:\n",
       "\tURI\tstr\n",
       "\tname\tstr\n",
       "\ttext\tstr\n",
       "\n",
       "Rows: 10\n",
       "\n",
       "Data:\n",
       "+-------------------------------+---------------------+\n",
       "|              URI              |         name        |\n",
       "+-------------------------------+---------------------+\n",
       "| <http://dbpedia.org/resour... |    Digby Morrell    |\n",
       "| <http://dbpedia.org/resour... |    Alfred J. Lewy   |\n",
       "| <http://dbpedia.org/resour... |    Harpdog Brown    |\n",
       "| <http://dbpedia.org/resour... | Franz Rottensteiner |\n",
       "| <http://dbpedia.org/resour... |        G-Enka       |\n",
       "| <http://dbpedia.org/resour... |    Sam Henderson    |\n",
       "| <http://dbpedia.org/resour... |    Aaron LaCrate    |\n",
       "| <http://dbpedia.org/resour... |   Trevor Ferguson   |\n",
       "| <http://dbpedia.org/resour... |     Grant Nelson    |\n",
       "| <http://dbpedia.org/resour... |     Cathy Caruth    |\n",
       "+-------------------------------+---------------------+\n",
       "+-------------------------------+\n",
       "|              text             |\n",
       "+-------------------------------+\n",
       "| digby morrell born 10 octo... |\n",
       "| alfred j lewy aka sandy le... |\n",
       "| harpdog brown is a singer ... |\n",
       "| franz rottensteiner born i... |\n",
       "| henry krvits born 30 decem... |\n",
       "| sam henderson born october... |\n",
       "| aaron lacrate is an americ... |\n",
       "| trevor ferguson aka john f... |\n",
       "| grant nelson born 27 april... |\n",
       "| cathy caruth born 1955 is ... |\n",
       "+-------------------------------+\n",
       "[10 rows x 3 columns]"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "people.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "john = people[people['name'] == 'Elton John']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\"><table frame=\"box\" rules=\"cols\">\n",
       "    <tr>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">URI</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">name</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">text</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">&lt;http://dbpedia.org/resou<br>rce/Elton_John&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Elton John</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">sir elton hercules john<br>cbe born reginald ken ...</td>\n",
       "    </tr>\n",
       "</table>\n",
       "[1 rows x 3 columns]<br/>\n",
       "</div>"
      ],
      "text/plain": [
       "Columns:\n",
       "\tURI\tstr\n",
       "\tname\tstr\n",
       "\ttext\tstr\n",
       "\n",
       "Rows: 1\n",
       "\n",
       "Data:\n",
       "+-------------------------------+------------+-------------------------------+\n",
       "|              URI              |    name    |              text             |\n",
       "+-------------------------------+------------+-------------------------------+\n",
       "| <http://dbpedia.org/resour... | Elton John | sir elton hercules john cb... |\n",
       "+-------------------------------+------------+-------------------------------+\n",
       "[1 rows x 3 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "john.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "john['wordCount'] = graphlab.text_analytics.count_words(john['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\"><table frame=\"box\" rules=\"cols\">\n",
       "    <tr>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">URI</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">name</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">text</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">wordCount</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">&lt;http://dbpedia.org/resou<br>rce/Elton_John&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Elton John</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">sir elton hercules john<br>cbe born reginald ken ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">{'all': 1, 'least': 1,<br>'producer': 1, 'heavi ...</td>\n",
       "    </tr>\n",
       "</table>\n",
       "[1 rows x 4 columns]<br/>\n",
       "</div>"
      ],
      "text/plain": [
       "Columns:\n",
       "\tURI\tstr\n",
       "\tname\tstr\n",
       "\ttext\tstr\n",
       "\twordCount\tdict\n",
       "\n",
       "Rows: 1\n",
       "\n",
       "Data:\n",
       "+-------------------------------+------------+-------------------------------+\n",
       "|              URI              |    name    |              text             |\n",
       "+-------------------------------+------------+-------------------------------+\n",
       "| <http://dbpedia.org/resour... | Elton John | sir elton hercules john cb... |\n",
       "+-------------------------------+------------+-------------------------------+\n",
       "+-------------------------------+\n",
       "|           wordCount           |\n",
       "+-------------------------------+\n",
       "| {'all': 1, 'least': 1, 'pr... |\n",
       "+-------------------------------+\n",
       "[1 rows x 4 columns]"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "john.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\"><table frame=\"box\" rules=\"cols\">\n",
       "    <tr>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">word</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">the</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">in</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">and</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">of</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">a</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">has</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">john</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">he</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">on</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">award</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">5</td>\n",
       "    </tr>\n",
       "</table>\n",
       "[255 rows x 2 columns]<br/>Note: Only the head of the SFrame is printed.<br/>You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.\n",
       "</div>"
      ],
      "text/plain": [
       "Columns:\n",
       "\tword\tstr\n",
       "\tcount\tint\n",
       "\n",
       "Rows: 255\n",
       "\n",
       "Data:\n",
       "+-------+-------+\n",
       "|  word | count |\n",
       "+-------+-------+\n",
       "|  the  |   27  |\n",
       "|   in  |   18  |\n",
       "|  and  |   15  |\n",
       "|   of  |   13  |\n",
       "|   a   |   10  |\n",
       "|  has  |   9   |\n",
       "|  john |   7   |\n",
       "|   he  |   7   |\n",
       "|   on  |   6   |\n",
       "| award |   5   |\n",
       "+-------+-------+\n",
       "[255 rows x 2 columns]\n",
       "Note: Only the head of the SFrame is printed.\n",
       "You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns."
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "john[['wordCount']].stack('wordCount', new_column_name=['word','count']).sort('count',ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "people['wordCount'] = graphlab.text_analytics.count_words(people['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf = graphlab.text_analytics.tf_idf(people['wordCount'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "people['tfidf'] = tfidf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\"><table frame=\"box\" rules=\"cols\">\n",
       "    <tr>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">URI</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">name</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">text</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">wordCount</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">&lt;http://dbpedia.org/resou<br>rce/Digby_Morrell&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Digby Morrell</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">digby morrell born 10<br>october 1979 is a former ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">{'selection': 1,<br>'carltons': 1, 'being': ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">&lt;http://dbpedia.org/resou<br>rce/Alfred_J._Lewy&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Alfred J. Lewy</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">alfred j lewy aka sandy<br>lewy graduated from ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">{'precise': 1, 'thomas':<br>1, 'closely': 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">&lt;http://dbpedia.org/resou<br>rce/Harpdog_Brown&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Harpdog Brown</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">harpdog brown is a singer<br>and harmonica player who ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">{'just': 1, 'issued': 1,<br>'mainly': 1, 'nominat ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">&lt;http://dbpedia.org/resou<br>rce/Franz_Rottensteiner&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Franz Rottensteiner</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">franz rottensteiner born<br>in waidmannsfeld lower ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">{'all': 1,<br>'bauforschung': 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">&lt;http://dbpedia.org/resou<br>rce/G-Enka&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">G-Enka</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">henry krvits born 30<br>december 1974 in tallinn ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">{'they': 1,<br>'gangstergenka': 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">&lt;http://dbpedia.org/resou<br>rce/Sam_Henderson&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Sam Henderson</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">sam henderson born<br>october 18 1969 is an ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">{'currently': 1, 'less':<br>1, 'being': 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">&lt;http://dbpedia.org/resou<br>rce/Aaron_LaCrate&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Aaron LaCrate</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">aaron lacrate is an<br>american music producer ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">{'exclusive': 2,<br>'producer': 1, 'show' ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">&lt;http://dbpedia.org/resou<br>rce/Trevor_Ferguson&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Trevor Ferguson</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">trevor ferguson aka john<br>farrow born 11 november ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">{'taxi': 1, 'salon': 1,<br>'gangs': 1, 'being': 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">&lt;http://dbpedia.org/resou<br>rce/Grant_Nelson&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Grant Nelson</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">grant nelson born 27<br>april 1971 in london  ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">{'houston': 1, 'frankie':<br>1, 'labels': 1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">&lt;http://dbpedia.org/resou<br>rce/Cathy_Caruth&gt; ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Cathy Caruth</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">cathy caruth born 1955 is<br>frank h t rhodes ...</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">{'phenomenon': 1,<br>'deborash': 1, 'both' ...</td>\n",
       "    </tr>\n",
       "</table>\n",
       "<table frame=\"box\" rules=\"cols\">\n",
       "    <tr>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">tfidf</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">{'selection':<br>3.836578553093086, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">{'precise':<br>6.44320060695519, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">{'just':<br>2.7007299687108643, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">{'all':<br>1.6431112434912472, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">{'they':<br>1.8993401178193898, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">{'currently':<br>1.637088969126014, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">{'exclusive':<br>10.455187230695827, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">{'taxi':<br>6.0520214560945025, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">{'houston':<br>3.935505942157149, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">{'phenomenon':<br>5.750053426395245, ...</td>\n",
       "    </tr>\n",
       "</table>\n",
       "[10 rows x 5 columns]<br/>\n",
       "</div>"
      ],
      "text/plain": [
       "Columns:\n",
       "\tURI\tstr\n",
       "\tname\tstr\n",
       "\ttext\tstr\n",
       "\twordCount\tdict\n",
       "\ttfidf\tdict\n",
       "\n",
       "Rows: 10\n",
       "\n",
       "Data:\n",
       "+-------------------------------+---------------------+\n",
       "|              URI              |         name        |\n",
       "+-------------------------------+---------------------+\n",
       "| <http://dbpedia.org/resour... |    Digby Morrell    |\n",
       "| <http://dbpedia.org/resour... |    Alfred J. Lewy   |\n",
       "| <http://dbpedia.org/resour... |    Harpdog Brown    |\n",
       "| <http://dbpedia.org/resour... | Franz Rottensteiner |\n",
       "| <http://dbpedia.org/resour... |        G-Enka       |\n",
       "| <http://dbpedia.org/resour... |    Sam Henderson    |\n",
       "| <http://dbpedia.org/resour... |    Aaron LaCrate    |\n",
       "| <http://dbpedia.org/resour... |   Trevor Ferguson   |\n",
       "| <http://dbpedia.org/resour... |     Grant Nelson    |\n",
       "| <http://dbpedia.org/resour... |     Cathy Caruth    |\n",
       "+-------------------------------+---------------------+\n",
       "+-------------------------------+-------------------------------+\n",
       "|              text             |           wordCount           |\n",
       "+-------------------------------+-------------------------------+\n",
       "| digby morrell born 10 octo... | {'selection': 1, 'carltons... |\n",
       "| alfred j lewy aka sandy le... | {'precise': 1, 'thomas': 1... |\n",
       "| harpdog brown is a singer ... | {'just': 1, 'issued': 1, '... |\n",
       "| franz rottensteiner born i... | {'all': 1, 'bauforschung':... |\n",
       "| henry krvits born 30 decem... | {'they': 1, 'gangstergenka... |\n",
       "| sam henderson born october... | {'currently': 1, 'less': 1... |\n",
       "| aaron lacrate is an americ... | {'exclusive': 2, 'producer... |\n",
       "| trevor ferguson aka john f... | {'taxi': 1, 'salon': 1, 'g... |\n",
       "| grant nelson born 27 april... | {'houston': 1, 'frankie': ... |\n",
       "| cathy caruth born 1955 is ... | {'phenomenon': 1, 'deboras... |\n",
       "+-------------------------------+-------------------------------+\n",
       "+-------------------------------+\n",
       "|             tfidf             |\n",
       "+-------------------------------+\n",
       "| {'selection': 3.8365785530... |\n",
       "| {'precise': 6.443200606955... |\n",
       "| {'just': 2.700729968710864... |\n",
       "| {'all': 1.6431112434912472... |\n",
       "| {'they': 1.899340117819389... |\n",
       "| {'currently': 1.6370889691... |\n",
       "| {'exclusive': 10.455187230... |\n",
       "| {'taxi': 6.052021456094502... |\n",
       "| {'houston': 3.935505942157... |\n",
       "| {'phenomenon': 5.750053426... |\n",
       "+-------------------------------+\n",
       "[10 rows x 5 columns]"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "people.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "john = people[people['name'] == 'Elton John']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\"><table frame=\"box\" rules=\"cols\">\n",
       "    <tr>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">word</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">furnish</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">18.38947184</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">elton</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">17.48232027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">billboard</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">17.3036809575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">john</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">13.9393127924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">songwriters</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">11.250406447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">tonightcandle</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">10.9864953892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">overallelton</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">10.9864953892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">19702000</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">10.2933482087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">fivedecade</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">10.2933482087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">aids</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">10.262846934</td>\n",
       "    </tr>\n",
       "</table>\n",
       "[255 rows x 2 columns]<br/>Note: Only the head of the SFrame is printed.<br/>You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns.\n",
       "</div>"
      ],
      "text/plain": [
       "Columns:\n",
       "\tword\tstr\n",
       "\tcount\tfloat\n",
       "\n",
       "Rows: 255\n",
       "\n",
       "Data:\n",
       "+---------------+---------------+\n",
       "|      word     |     count     |\n",
       "+---------------+---------------+\n",
       "|    furnish    |  18.38947184  |\n",
       "|     elton     |  17.48232027  |\n",
       "|   billboard   | 17.3036809575 |\n",
       "|      john     | 13.9393127924 |\n",
       "|  songwriters  |  11.250406447 |\n",
       "| tonightcandle | 10.9864953892 |\n",
       "|  overallelton | 10.9864953892 |\n",
       "|    19702000   | 10.2933482087 |\n",
       "|   fivedecade  | 10.2933482087 |\n",
       "|      aids     |  10.262846934 |\n",
       "+---------------+---------------+\n",
       "[255 rows x 2 columns]\n",
       "Note: Only the head of the SFrame is printed.\n",
       "You can use print_rows(num_rows=m, num_columns=n) to print more rows and columns."
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "john[['tfidf']].stack('tfidf',new_column_name=['word','count']).sort('count',ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "beckham = people[people['name']=='Victoria Beckham']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9567006376655429"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "graphlab.distances.cosine(john['tfidf'][0],beckham['tfidf'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "paul = people[people['name']=='Paul McCartney']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8250310029221779"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "graphlab.distances.cosine(john['tfidf'][0],paul['tfidf'][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Comparing the difference in clustering with tf-idf than to raw word count "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre>Starting brute force nearest neighbors model training.</pre>"
      ],
      "text/plain": [
       "Starting brute force nearest neighbors model training."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "tdidfKnnModel = graphlab.nearest_neighbors.create(people,features=['tfidf'],label='name',distance='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre>Starting brute force nearest neighbors model training.</pre>"
      ],
      "text/plain": [
       "Starting brute force nearest neighbors model training."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "wordCountKnnModel = graphlab.nearest_neighbors.create(people,features=['wordCount'],label='name',distance='cosine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre>Starting pairwise querying.</pre>"
      ],
      "text/plain": [
       "Starting pairwise querying."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+--------------+---------+-------------+--------------+</pre>"
      ],
      "text/plain": [
       "+--------------+---------+-------------+--------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| Query points | # Pairs | % Complete. | Elapsed Time |</pre>"
      ],
      "text/plain": [
       "| Query points | # Pairs | % Complete. | Elapsed Time |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+--------------+---------+-------------+--------------+</pre>"
      ],
      "text/plain": [
       "+--------------+---------+-------------+--------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 0            | 1       | 0.00169288  | 24.204ms     |</pre>"
      ],
      "text/plain": [
       "| 0            | 1       | 0.00169288  | 24.204ms     |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| Done         |         | 100         | 447.543ms    |</pre>"
      ],
      "text/plain": [
       "| Done         |         | 100         | 447.543ms    |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+--------------+---------+-------------+--------------+</pre>"
      ],
      "text/plain": [
       "+--------------+---------+-------------+--------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\"><table frame=\"box\" rules=\"cols\">\n",
       "    <tr>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">query_label</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">reference_label</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">distance</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">rank</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Elton John</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">2.22044604925e-16</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Cliff Richard</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0.16142415259</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Sandro Petrone</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0.16822542751</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Rod Stewart</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0.168327165587</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Malachi O'Doherty</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0.177315545979</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">5</td>\n",
       "    </tr>\n",
       "</table>\n",
       "[5 rows x 4 columns]<br/>\n",
       "</div>"
      ],
      "text/plain": [
       "Columns:\n",
       "\tquery_label\tint\n",
       "\treference_label\tstr\n",
       "\tdistance\tfloat\n",
       "\trank\tint\n",
       "\n",
       "Rows: 5\n",
       "\n",
       "Data:\n",
       "+-------------+-------------------+-------------------+------+\n",
       "| query_label |  reference_label  |      distance     | rank |\n",
       "+-------------+-------------------+-------------------+------+\n",
       "|      0      |     Elton John    | 2.22044604925e-16 |  1   |\n",
       "|      0      |   Cliff Richard   |   0.16142415259   |  2   |\n",
       "|      0      |   Sandro Petrone  |   0.16822542751   |  3   |\n",
       "|      0      |    Rod Stewart    |   0.168327165587  |  4   |\n",
       "|      0      | Malachi O'Doherty |   0.177315545979  |  5   |\n",
       "+-------------+-------------------+-------------------+------+\n",
       "[5 rows x 4 columns]"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordCountKnnModel.query(john)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre>Starting pairwise querying.</pre>"
      ],
      "text/plain": [
       "Starting pairwise querying."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+--------------+---------+-------------+--------------+</pre>"
      ],
      "text/plain": [
       "+--------------+---------+-------------+--------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| Query points | # Pairs | % Complete. | Elapsed Time |</pre>"
      ],
      "text/plain": [
       "| Query points | # Pairs | % Complete. | Elapsed Time |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+--------------+---------+-------------+--------------+</pre>"
      ],
      "text/plain": [
       "+--------------+---------+-------------+--------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 0            | 1       | 0.00169288  | 21.838ms     |</pre>"
      ],
      "text/plain": [
       "| 0            | 1       | 0.00169288  | 21.838ms     |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| Done         |         | 100         | 530.075ms    |</pre>"
      ],
      "text/plain": [
       "| Done         |         | 100         | 530.075ms    |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+--------------+---------+-------------+--------------+</pre>"
      ],
      "text/plain": [
       "+--------------+---------+-------------+--------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\"><table frame=\"box\" rules=\"cols\">\n",
       "    <tr>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">query_label</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">reference_label</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">distance</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">rank</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Elton John</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">-2.22044604925e-16</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Rod Stewart</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0.717219667893</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">George Michael</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0.747600998969</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Sting (musician)</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0.747671954431</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Phil Collins</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0.75119324879</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">5</td>\n",
       "    </tr>\n",
       "</table>\n",
       "[5 rows x 4 columns]<br/>\n",
       "</div>"
      ],
      "text/plain": [
       "Columns:\n",
       "\tquery_label\tint\n",
       "\treference_label\tstr\n",
       "\tdistance\tfloat\n",
       "\trank\tint\n",
       "\n",
       "Rows: 5\n",
       "\n",
       "Data:\n",
       "+-------------+------------------+--------------------+------+\n",
       "| query_label | reference_label  |      distance      | rank |\n",
       "+-------------+------------------+--------------------+------+\n",
       "|      0      |    Elton John    | -2.22044604925e-16 |  1   |\n",
       "|      0      |   Rod Stewart    |   0.717219667893   |  2   |\n",
       "|      0      |  George Michael  |   0.747600998969   |  3   |\n",
       "|      0      | Sting (musician) |   0.747671954431   |  4   |\n",
       "|      0      |   Phil Collins   |   0.75119324879    |  5   |\n",
       "+-------------+------------------+--------------------+------+\n",
       "[5 rows x 4 columns]"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tdidfKnnModel.query(john)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre>Starting pairwise querying.</pre>"
      ],
      "text/plain": [
       "Starting pairwise querying."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+--------------+---------+-------------+--------------+</pre>"
      ],
      "text/plain": [
       "+--------------+---------+-------------+--------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| Query points | # Pairs | % Complete. | Elapsed Time |</pre>"
      ],
      "text/plain": [
       "| Query points | # Pairs | % Complete. | Elapsed Time |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+--------------+---------+-------------+--------------+</pre>"
      ],
      "text/plain": [
       "+--------------+---------+-------------+--------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 0            | 1       | 0.00169288  | 17.312ms     |</pre>"
      ],
      "text/plain": [
       "| 0            | 1       | 0.00169288  | 17.312ms     |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| Done         |         | 100         | 660.509ms    |</pre>"
      ],
      "text/plain": [
       "| Done         |         | 100         | 660.509ms    |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+--------------+---------+-------------+--------------+</pre>"
      ],
      "text/plain": [
       "+--------------+---------+-------------+--------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\"><table frame=\"box\" rules=\"cols\">\n",
       "    <tr>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">query_label</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">reference_label</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">distance</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">rank</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Victoria Beckham</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">-2.22044604925e-16</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Mary Fitzgerald (artist)</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0.207307036115</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Adrienne Corri</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0.214509782788</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Beverly Jane Fry</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0.217466468741</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Raman Mundair</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0.217695474992</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">5</td>\n",
       "    </tr>\n",
       "</table>\n",
       "[5 rows x 4 columns]<br/>\n",
       "</div>"
      ],
      "text/plain": [
       "Columns:\n",
       "\tquery_label\tint\n",
       "\treference_label\tstr\n",
       "\tdistance\tfloat\n",
       "\trank\tint\n",
       "\n",
       "Rows: 5\n",
       "\n",
       "Data:\n",
       "+-------------+--------------------------+--------------------+------+\n",
       "| query_label |     reference_label      |      distance      | rank |\n",
       "+-------------+--------------------------+--------------------+------+\n",
       "|      0      |     Victoria Beckham     | -2.22044604925e-16 |  1   |\n",
       "|      0      | Mary Fitzgerald (artist) |   0.207307036115   |  2   |\n",
       "|      0      |      Adrienne Corri      |   0.214509782788   |  3   |\n",
       "|      0      |     Beverly Jane Fry     |   0.217466468741   |  4   |\n",
       "|      0      |      Raman Mundair       |   0.217695474992   |  5   |\n",
       "+-------------+--------------------------+--------------------+------+\n",
       "[5 rows x 4 columns]"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordCountKnnModel.query(beckham)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre>Starting pairwise querying.</pre>"
      ],
      "text/plain": [
       "Starting pairwise querying."
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+--------------+---------+-------------+--------------+</pre>"
      ],
      "text/plain": [
       "+--------------+---------+-------------+--------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| Query points | # Pairs | % Complete. | Elapsed Time |</pre>"
      ],
      "text/plain": [
       "| Query points | # Pairs | % Complete. | Elapsed Time |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+--------------+---------+-------------+--------------+</pre>"
      ],
      "text/plain": [
       "+--------------+---------+-------------+--------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| 0            | 1       | 0.00169288  | 24.838ms     |</pre>"
      ],
      "text/plain": [
       "| 0            | 1       | 0.00169288  | 24.838ms     |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>| Done         |         | 100         | 885.843ms    |</pre>"
      ],
      "text/plain": [
       "| Done         |         | 100         | 885.843ms    |"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre>+--------------+---------+-------------+--------------+</pre>"
      ],
      "text/plain": [
       "+--------------+---------+-------------+--------------+"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\"><table frame=\"box\" rules=\"cols\">\n",
       "    <tr>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">query_label</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">reference_label</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">distance</th>\n",
       "        <th style=\"padding-left: 1em; padding-right: 1em; text-align: center\">rank</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Victoria Beckham</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">1.11022302463e-16</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">David Beckham</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0.548169610263</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Stephen Dow Beckham</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0.784986706828</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Mel B</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0.809585523409</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">Caroline Rush</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">0.819826422919</td>\n",
       "        <td style=\"padding-left: 1em; padding-right: 1em; text-align: center; vertical-align: top\">5</td>\n",
       "    </tr>\n",
       "</table>\n",
       "[5 rows x 4 columns]<br/>\n",
       "</div>"
      ],
      "text/plain": [
       "Columns:\n",
       "\tquery_label\tint\n",
       "\treference_label\tstr\n",
       "\tdistance\tfloat\n",
       "\trank\tint\n",
       "\n",
       "Rows: 5\n",
       "\n",
       "Data:\n",
       "+-------------+---------------------+-------------------+------+\n",
       "| query_label |   reference_label   |      distance     | rank |\n",
       "+-------------+---------------------+-------------------+------+\n",
       "|      0      |   Victoria Beckham  | 1.11022302463e-16 |  1   |\n",
       "|      0      |    David Beckham    |   0.548169610263  |  2   |\n",
       "|      0      | Stephen Dow Beckham |   0.784986706828  |  3   |\n",
       "|      0      |        Mel B        |   0.809585523409  |  4   |\n",
       "|      0      |    Caroline Rush    |   0.819826422919  |  5   |\n",
       "+-------------+---------------------+-------------------+------+\n",
       "[5 rows x 4 columns]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tdidfKnnModel.query(beckham)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}