Music_Recommender_System.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Id0nHfqKPeHi"
   },
   "outputs": [],
   "source": [
    "!pip install lightfm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "BBjCABJuOqX8"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from scipy import sparse as sp\n",
    "from warnings import filterwarnings\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "from lightfm import LightFM\n",
    "from sklearn.metrics import pairwise as pw\n",
    "from lightfm.evaluation import precision_at_k, recall_at_k, auc_score, reciprocal_rank\n",
    "\n",
    "sns.set()\n",
    "filterwarnings('ignore')\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "J4wPYMsXQG4G"
   },
   "source": [
    "## Classes and UDFs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "zKv3VoVeS4s7"
   },
   "source": [
    "#### Useful functions for detailed data inspection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "of4r0RIJoqJs"
   },
   "outputs": [],
   "source": [
    "# Create Data audit Report for continuous variables\n",
    "def cont_summary(x):\n",
    "    return pd.Series([x.count(), x.isnull().sum(), x.sum(), x.mean(), x.median(),  \n",
    "                      x.std(), x.var(), x.min(), x.quantile(0.01), x.quantile(0.05),\n",
    "                          x.quantile(0.10),x.quantile(0.25),x.quantile(0.50),x.quantile(0.75), \n",
    "                              x.quantile(0.90),x.quantile(0.95), x.quantile(0.99),x.max()], \n",
    "                  index = ['N', 'NMISS', 'SUM', 'MEAN','MEDIAN', 'STD', 'VAR', 'MIN', 'P1', \n",
    "                               'P5' ,'P10' ,'P25' ,'P50' ,'P75' ,'P90' ,'P95' ,'P99' ,'MAX'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "3n_n2p--orW5"
   },
   "outputs": [],
   "source": [
    "# Create Data audit Report for categorical variables\n",
    "def cat_summary(x):\n",
    "    Mode = x.value_counts().sort_values(ascending = False)[0:1].reset_index()\n",
    "    return pd.Series([x.count(), x.isnull().sum(), Mode.iloc[0, 0], Mode.iloc[0, 1], \n",
    "                          round(Mode.iloc[0, 1] * 100/x.count(), 2)], \n",
    "                  index = ['N', 'NMISS', 'MODE', 'FREQ', 'PERCENT'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "afk9gJxYqK6G"
   },
   "outputs": [],
   "source": [
    "# Function to return key for any value\n",
    "def get_key(val, dictionary):\n",
    "    for key, value in dictionary.items():\n",
    "        if val == value:\n",
    "            return key\n",
    " \n",
    "    raise Exception(\"Song doesn't exist in the database!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "OTCuXNkwPou3"
   },
   "source": [
    "#### Recommendation Class for popularity based model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "R3KnjPr7Plfb"
   },
   "outputs": [],
   "source": [
    "# Class for Popularity based Recommender System model\n",
    "class popularity_recommender():\n",
    "    def __init__(self):\n",
    "        self.train_data = None\n",
    "        self.user_id = None\n",
    "        self.item_id = None\n",
    "        self.popularity_recommendations = None\n",
    "        \n",
    "    # Create the popularity based recommender system model\n",
    "    def create(self, train_data, user_id, item_id):\n",
    "        self.train_data = train_data\n",
    "        self.user_id = user_id\n",
    "        self.item_id = item_id\n",
    "\n",
    "        # Get a count of user_ids for each unique song as recommendation score\n",
    "        train_data_grouped = train_data.groupby([self.item_id]).agg({self.user_id: 'count'}).reset_index()\n",
    "        train_data_grouped.rename(columns = {'user_id': 'score'},inplace=True)\n",
    "    \n",
    "        # Sort the songs based upon recommendation score\n",
    "        train_data_sort = train_data_grouped.sort_values(['score', self.item_id], ascending = [0,1])\n",
    "    \n",
    "        # Generate a recommendation rank based upon score\n",
    "        train_data_sort['Rank'] = train_data_sort['score'].rank(ascending=0, method='first')\n",
    "        \n",
    "        # Get the top 10 recommendations\n",
    "        self.popularity_recommendations = train_data_sort.head(10)\n",
    "\n",
    "    # Use the popularity based recommender system model to make recommendations\n",
    "    def recommend(self, user_id):    \n",
    "        user_recommendations = self.popularity_recommendations\n",
    "        \n",
    "        # Add user_id column for which the recommendations are being generated\n",
    "        user_recommendations['user_id'] = user_id\n",
    "    \n",
    "        # Bring user_id column to the front\n",
    "        cols = user_recommendations.columns.tolist()\n",
    "        cols = cols[-1:] + cols[:-1]\n",
    "        user_recommendations = user_recommendations[cols]\n",
    "        \n",
    "        return user_recommendations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "OJR1RAPdQRDh"
   },
   "source": [
    "#### Useful functions for personalized hybrid recommendation model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "DYbnDnugQQNG"
   },
   "outputs": [],
   "source": [
    "# Function to create a user dictionary based on their index and number in interaction dataset\n",
    "def create_user_dict(interactions):\n",
    "    user_id = list(interactions.index)\n",
    "    user_dict = {}\n",
    "    counter = 0 \n",
    "\n",
    "    for i in user_id:\n",
    "        user_dict[i] = counter\n",
    "        counter += 1\n",
    "\n",
    "    new_dict = dict([(value, key) for key, value in user_dict.items()])\n",
    "\n",
    "    return new_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "8sbBs3wOQbPn"
   },
   "outputs": [],
   "source": [
    "# Function to create an item dictionary based on their item_id and item name  \n",
    "def create_item_dict(df, id_col, name_col):\n",
    "    item_dict ={}\n",
    "\n",
    "    for i in range(df.shape[0]):\n",
    "        item_dict[(df.loc[i, id_col])] = df.loc[i, name_col]\n",
    "\n",
    "    return item_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "1lsLgzFrQe_N"
   },
   "outputs": [],
   "source": [
    "# Function to produce user recommendations\n",
    "def sample_recommendation_user(model, interactions, user_id, user_dict, \n",
    "                               item_dict, threshold = 0, nrec_items = 10):\n",
    "    n_users, n_items = interactions.shape\n",
    "    user_x = user_dict[user_id]\n",
    "    scores = pd.Series(model.predict(user_x, np.arange(n_items)))\n",
    "    scores.index = interactions.columns\n",
    "    scores = list(pd.Series(scores.sort_values(ascending=False).index))\n",
    "    \n",
    "    known_items = list(pd.Series(interactions.loc[user_id, :] \\\n",
    "                                 [interactions.loc[user_id, :] > threshold].index) \\\n",
    "                                  .sort_values(ascending=False))\n",
    "    \n",
    "    scores = [x for x in scores if x not in known_items]\n",
    "    return_score_list = scores[0: nrec_items]\n",
    "    known_items = list(pd.Series(known_items).apply(lambda x: item_dict[x]))\n",
    "    scores = list(pd.Series(return_score_list).apply(lambda x: item_dict[x]))\n",
    "    \n",
    "    print(\"Recommended songs for UserID:\", user_id)\n",
    "    counter = 1\n",
    "\n",
    "    for i in scores:\n",
    "        print(str(counter) + '- ' + i)\n",
    "        counter+=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2cv6xpKTQhsC"
   },
   "outputs": [],
   "source": [
    "# Function to create item-item distance embedding matrix\n",
    "def create_item_emdedding_distance_matrix(model, interactions):\n",
    "    \n",
    "    df_item_norm_sparse = sp.csr_matrix(model.item_embeddings)\n",
    "    similarities = pw.cosine_similarity(df_item_norm_sparse)\n",
    "    item_emdedding_distance_matrix = pd.DataFrame(similarities)\n",
    "    item_emdedding_distance_matrix.columns = interactions.columns\n",
    "    item_emdedding_distance_matrix.index = interactions.columns\n",
    "    \n",
    "    return item_emdedding_distance_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "d48nez6SQj4v"
   },
   "outputs": [],
   "source": [
    "# Function to create item-item recommendation\n",
    "def item_item_recommendation(item_emdedding_distance_matrix, item_id, item_dict, n_items = 10):\n",
    "    \n",
    "    recommended_items = list(pd.Series(item_emdedding_distance_matrix.loc[item_id,:]. \\\n",
    "                                  sort_values(ascending = False).head(n_items+1). \\\n",
    "                                  index[1:n_items+1]))\n",
    "    \n",
    "    print(\"Song of interest: {0}\".format(item_dict[item_id]))\n",
    "    print(\"Song(s) similar to the above item are as follows:-\")\n",
    "    counter = 1\n",
    "    \n",
    "    for i in recommended_items:\n",
    "        print(str(counter) + '. ' +  item_dict[i])\n",
    "        counter+=1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3vkfzvvOQ4TN"
   },
   "source": [
    "## Data Import & Inspection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XHd_pbpvQkXu"
   },
   "outputs": [],
   "source": [
    "triplets = '10000.txt'\n",
    "songsData = 'song_data.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "baKZzeawQ9iw"
   },
   "outputs": [],
   "source": [
    "rawData1 = pd.read_table(triplets, header=None)\n",
    "rawData1.columns = ['user_id', 'song_id', 'listen_count']\n",
    "rawData2 =  pd.read_csv(songsData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "McV-rSzMQ-C4"
   },
   "outputs": [],
   "source": [
    "# Create a new copy of the triplets dataset & change user_ids\n",
    "# from string format to indexed values for easier computations\n",
    "rawData1_userIndexed = rawData1.copy()\n",
    "rawData1_userIndexed.user_id = rawData1.index + 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "qn_pW2MIRPWB"
   },
   "outputs": [],
   "source": [
    "# Merge the triplets data (user indexed) with songs data\n",
    "rawData = pd.merge(rawData1_userIndexed, rawData2.drop_duplicates(['song_id']), on=\"song_id\", how=\"left\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "08U3n1EQTUjj"
   },
   "outputs": [],
   "source": [
    "# Create a subset of top fifty thousand observations to work with, \n",
    "# as the entire dataset is TOO expensive to compute on!!!\n",
    "data = rawData.head(50000).copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "6T2QdXbIR6Et"
   },
   "outputs": [],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "BxroaKWERPvK"
   },
   "outputs": [],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "acNL3NfaR4LW"
   },
   "outputs": [],
   "source": [
    "data.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YZPt43hFR5Xn"
   },
   "outputs": [],
   "source": [
    "data.describe(include='all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "X6Q9LtmOSYVP"
   },
   "outputs": [],
   "source": [
    "data.select_dtypes(include='int64').apply(cont_summary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "CAPvoY4VTA0I"
   },
   "outputs": [],
   "source": [
    "data.select_dtypes(include='object').apply(cat_summary)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6dz1018WUqWI"
   },
   "source": [
    "## EDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "AKcPFphZTD16"
   },
   "outputs": [],
   "source": [
    "grp_title = rawData.groupby(['title']).agg({'listen_count': 'count'}).reset_index()\n",
    "grp_title['percentage'] = rawData['listen_count'].div(rawData.listen_count.sum()) * 100\n",
    "grp_title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "p4t-Ia7HW-AB"
   },
   "outputs": [],
   "source": [
    "grp_title.sort_values(by=['listen_count']).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "FXr6v0aJUukM"
   },
   "outputs": [],
   "source": [
    "grp_title.sort_values(by=['listen_count'], ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "tiNceiqHUwV_"
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize=(7, 4), dpi=110)\n",
    "plt.hist(grp_title.listen_count, bins=150)\n",
    "plt.xlim(0, 800)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hA9zmkb0XS-F"
   },
   "source": [
    "## Recommedations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "PVO8nQ2rU5qi"
   },
   "outputs": [],
   "source": [
    "users = data['user_id'].unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "q5vCe-CzZa5h"
   },
   "source": [
    "#### Interaction Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ok5EzQdoXzV1"
   },
   "outputs": [],
   "source": [
    "# Create a pivot table (interaction matrix) from the original dataset\n",
    "x = data.pivot_table(index='user_id', columns='song_id', values='listen_count')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "up5tVmB8aupM"
   },
   "outputs": [],
   "source": [
    "xNan = x.fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_fZuO_C1aubF"
   },
   "outputs": [],
   "source": [
    "interaction = sp.csr_matrix(xNan.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mIRjoq2IXhuf"
   },
   "source": [
    "#### Popularity Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "MiCNBv1lYYq9"
   },
   "outputs": [],
   "source": [
    "id = int(input('Enter the ID of a user to get their popularity-based song recommendations: '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ifzd8UUkXZap"
   },
   "outputs": [],
   "source": [
    "popModel = popularity_recommender()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "FGbT8oJ5Xn5h"
   },
   "outputs": [],
   "source": [
    "# Popularity based recommendations by title\n",
    "popModel.create(data, 'user_id', 'title')\n",
    "popModel.recommend(users[id-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "z3Dgv8a-XpfI"
   },
   "outputs": [],
   "source": [
    "# Popularity based recommendations by artists\n",
    "popModel.create(data, 'user_id', 'artist_name')\n",
    "popModel.recommend(users[id-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vtBbZMzcbpLY"
   },
   "source": [
    "#### Personalized Hybrid Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "xHtkpCUrbDDi"
   },
   "outputs": [],
   "source": [
    "hybridModel = LightFM(loss='warp-kos', n=20, k=20, learning_schedule='adadelta')\n",
    "hybridModel.fit(interaction, epochs=600, num_threads=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "HpEknOgmbt5O"
   },
   "outputs": [],
   "source": [
    "precision_at_k(hybridModel, interaction).mean().round(4) * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "LWOoj3tDb8Aw"
   },
   "outputs": [],
   "source": [
    "recall_at_k(hybridModel, interaction).mean().round(4) * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "HP_54MN1b90w"
   },
   "outputs": [],
   "source": [
    "auc_score(hybridModel, interaction).mean().round(4) * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Fxek2OPUb_QU"
   },
   "outputs": [],
   "source": [
    "reciprocal_rank(hybridModel, interaction).mean().round(4) * 100"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "xR7btmyre6CY"
   },
   "source": [
    "#### Personal Recommendations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "FGKJQlapea8T"
   },
   "outputs": [],
   "source": [
    "# Creating user dictionary based on their index and number in the interaction matrix using recsys library\n",
    "userDict = create_user_dict(interactions=x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "pTIJ_GgUfRJ2"
   },
   "outputs": [],
   "source": [
    "# Creating a song dictionary based on their songID and artist name\n",
    "songDict = create_item_dict(df=rawData, id_col='song_id', name_col='title')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_RSEGYX9fT4k"
   },
   "outputs": [],
   "source": [
    "# Recommend songs using lightfm library\n",
    "id = int(input('Enter the ID of a user to get their personalized song recommendations: '))\n",
    "sample_recommendation_user(model=hybridModel, interactions=x, user_id=id, \n",
    "                           user_dict=userDict, item_dict=songDict, threshold=5, nrec_items=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "NcsSVWgKfVqV"
   },
   "outputs": [],
   "source": [
    "# Recommend songs similar to a given songID\n",
    "song = input('Enter a song to get similar recommendations: ')\n",
    "songID = get_key(song, songDict)\n",
    "songItemDist = create_item_emdedding_distance_matrix(model=hybridModel, interactions=x)\n",
    "item_item_recommendation(item_emdedding_distance_matrix=songItemDist, item_id=songID,\n",
    "                                    item_dict=songDict, n_items=10)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "Music Recommender System.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}