sp20-bcs-023_ids-A5.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8513f8ce",
   "metadata": {},
   "source": [
    "## MINAHIL SADIQ\n",
    "### SP20-BCS-023"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2026780d",
   "metadata": {},
   "source": [
    "### QUESTION 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "4037dfc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#importing libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from numpy.linalg import norm\n",
    "import math\n",
    "import warnings\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3c799e30",
   "metadata": {},
   "outputs": [],
   "source": [
    "#ignoring warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "d8bb2096",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['sunshine state enjoy sunshine',\n",
       " 'brown fox jump high, brown fox run',\n",
       " 'sunshine state fox run fast']"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# three documents in single list\n",
    "S1 = \"sunshine state enjoy sunshine\"\n",
    "S2 = \"brown fox jump high, brown fox run\"\n",
    "S3 = \"sunshine state fox run fast\"\n",
    "\n",
    "phrases = [S1, S2, S3]\n",
    "phrases"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "cc18b3cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "#bag of words\n",
    "#term frequency\n",
    "vect = CountVectorizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "c42e5828",
   "metadata": {},
   "outputs": [],
   "source": [
    "bog = vect.fit_transform(phrases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "e7cc2a4e",
   "metadata": {},
   "outputs": [],
   "source": [
    "features = vect.get_feature_names_out()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "5efe5b55",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   brown  enjoy  fast  fox  high  jump  run  state  sunshine\n",
      "0      0      1     0    0     0     0    0      1         2\n",
      "1      2      0     0    2     1     1    1      0         0\n",
      "2      0      0     1    1     0     0    1      1         1\n"
     ]
    }
   ],
   "source": [
    "tf = pd.DataFrame(bog.toarray(), columns=features)\n",
    "print(tf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "3f35b6e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#term frequency- inverse document frequency\n",
    "tfidf = TfidfTransformer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "8d90740a",
   "metadata": {},
   "outputs": [],
   "source": [
    "xy = tfidf.fit_transform(bog)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "f9f9a6e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>brown</th>\n",
       "      <th>enjoy</th>\n",
       "      <th>fast</th>\n",
       "      <th>fox</th>\n",
       "      <th>high</th>\n",
       "      <th>jump</th>\n",
       "      <th>run</th>\n",
       "      <th>state</th>\n",
       "      <th>sunshine</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.50689</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.385503</td>\n",
       "      <td>0.771006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.670703</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.510087</td>\n",
       "      <td>0.335352</td>\n",
       "      <td>0.335352</td>\n",
       "      <td>0.255044</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.549351</td>\n",
       "      <td>0.417796</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.417796</td>\n",
       "      <td>0.417796</td>\n",
       "      <td>0.417796</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      brown    enjoy      fast       fox      high      jump       run  \\\n",
       "0  0.000000  0.50689  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
       "1  0.670703  0.00000  0.000000  0.510087  0.335352  0.335352  0.255044   \n",
       "2  0.000000  0.00000  0.549351  0.417796  0.000000  0.000000  0.417796   \n",
       "\n",
       "      state  sunshine  \n",
       "0  0.385503  0.771006  \n",
       "1  0.000000  0.000000  \n",
       "2  0.417796  0.417796  "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "matrix = xy.todense()\n",
    "list = matrix.tolist()\n",
    "tfidf_df = pd.DataFrame(list, columns=features)\n",
    "tfidf_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8bd1ef33",
   "metadata": {},
   "source": [
    "#### NOTE: The values did not match the values I get by manually solving the question, I tried to do it but did not find the correct way to handle this problem, But I have come across one website which says that \"sklearn's TfidfVectorizer{TfidfTransformer} uses a different formula from the standard one which is normally either idf(t) = log [ n / df(t) ] or idf(t) = log [ n / (df(t) + 1) ] (denominator is adjusted to prevent zero divisions, if a term is not in the corpus)."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b037e6c9",
   "metadata": {},
   "source": [
    "### QUESTION 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "281a934e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "6eded602",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['sunshine state enjoy sunshine', 'sunshine state fox run fast']"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "phrase = [S1 , S3]\n",
    "phrase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "38f1f7c9",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = vect.fit_transform(phrase).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "6ff6541f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.        , 0.54772256],\n",
       "       [0.54772256, 1.        ]])"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cos = cosine_similarity([X[0, :], X[1, :]])\n",
    "cos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "230c6793",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}