SeER.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "SeER",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "machine_shape": "hm",
      "mount_file_id": "1-w7I9-9EcKxF8aHaSdNHAliBA83zoGcU",
      "authorship_tag": "ABX9TyNOr6NTQNqcjvO1/DA4OsOM",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/kartikey-singh/Replicating-SeER/blob/master/SeER.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LGdQcRJVZE1M",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Switching to HIGH RAM Session, Only run when in 12GB Session\n",
        "a = []\n",
        "while(1):\n",
        "    a.append('1')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_FIPzEUhlj_2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# TODO\n",
        "# Model testing by running for multiple epochs.\n",
        "# NDCG, MAP@K scores \n",
        "# Implementing attention instead of LSTM.\n",
        "# Explainabilty part of the paper."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lw7Z6FAiaHbU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import json\n",
        "import requests  \n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import pickle\n",
        "from sklearn import preprocessing\n",
        "import h5py\n",
        "import time\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "\n",
        "from pathlib import Path\n",
        "from scipy.sparse import csr_matrix\n",
        "from sklearn.preprocessing import MaxAbsScaler"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k0_J7v-DaIvw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QxUbDa2gaJy_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "d8f66daa-73de-4542-fd20-75a6b1ea2269"
      },
      "source": [
        "# Change it for SeER\n",
        "PATH = \"/content/drive/My Drive/dataset/\"\n",
        "PATH_MODEL = \"/content/drive/My Drive/dataset/seer/\"\n",
        "try:  \n",
        "  os.mkdir(PATH_MODEL)\n",
        "  os.mkdir(PATH)\n",
        "except:\n",
        "  print(\"Files already exists\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Files already exists\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5QDiKiT2aK4Z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Loading lookups.\n",
        "\n",
        "with open(PATH + 'tracks_sample.pkl', 'rb') as fp:\n",
        "    songs_lookup = pickle.load(fp)  \n",
        "\n",
        "with open(PATH + 'user_to_id.json', 'r') as f:\n",
        "    user_to_id = json.load(f)\n",
        "\n",
        "with open(PATH + 'track_to_id.json', 'r') as f:\n",
        "    track_to_id = json.load(f)        \n",
        "\n",
        "with open(PATH + 'id_to_track.json', 'r') as f:\n",
        "    id_to_track = json.load(f)            \n",
        "\n",
        "with open(PATH + 'id_to_user.json', 'r') as f:\n",
        "    id_to_user = json.load(f)                "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZgbP43QIahxv",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# So, median is 2600. So, keep at max 2600*32 values. \n",
        "# Padding smaller sequences with 0 and cutting larger ones.\n",
        "\n",
        "SONG_LEN = 2600*32\n",
        "for name, song in songs_lookup.items():\n",
        "    if song.shape[0] >= SONG_LEN:\n",
        "        songs_lookup[name] = song[:SONG_LEN]\n",
        "    else:\n",
        "        song_reference = np.zeros((SONG_LEN, ))        \n",
        "        song_reference[:song.shape[0]] = song   \n",
        "        songs_lookup[name] = song_reference"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ou1JpM1tajQB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 222
        },
        "outputId": "81e30e65-6469-4777-b95e-40153e64db31"
      },
      "source": [
        "df = pd.read_csv(PATH + 'df_1k.csv')\n",
        "num_users, num_tracks = df['user_id'].unique().shape[0], df['track_id'].unique().shape[0]\n",
        "\n",
        "# As id's start from 1 and not 0\n",
        "num_users = num_users + 1\n",
        "num_tracks = num_tracks + 1\n",
        "print(num_users, num_tracks, df.shape)\n",
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "32127 1001 (888373, 3)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>user_id</th>\n",
              "      <th>track_id</th>\n",
              "      <th>rating</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   user_id  track_id  rating\n",
              "0        1         1       1\n",
              "1        1         2       1\n",
              "2        1         3       5\n",
              "3        1         4       5\n",
              "4        1         5       5"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "heYec-8zam89",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_df = df.sample(frac=0.8, random_state=0)\n",
        "test_df = df.drop(train_df.index)\n",
        "\n",
        "train_rating = train_df.pop('rating')\n",
        "test_rating = test_df.pop('rating')\n",
        "\n",
        "train_dataset = tf.data.Dataset.from_tensor_slices((train_df.values, train_rating.values))\n",
        "test_dataset = tf.data.Dataset.from_tensor_slices((test_df.values, test_rating.values))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TPm8sVrcaoaD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# GLOBAL VARIABLES\n",
        "# Here buffer size is whole dataset.\n",
        "BUFFER_SIZE = len(train_df)\n",
        "BATCH_SIZE = 500\n",
        "EMBEDDING_SIZE = 50\n",
        "EPOCHS = 10\n",
        "steps_per_epoch = len(train_df)//BATCH_SIZE"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J15adK-FasYU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_dataset = train_dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K7X_nrlQavu2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# create_songs_batch functionality:\n",
        "# Here we first lookup for each song its MIDI data using JSON files\n",
        "def create_songs_batch(input_batch):\n",
        "    # No need to expand_dims as stacking does that for us.\n",
        "    # 32 is number of channels\n",
        "    songs_to_batch = []\n",
        "    for user_track_tensor in input_batch:\n",
        "        song_numpy = songs_lookup[id_to_track[str(user_track_tensor[1].numpy())]].reshape((-1, 32))\n",
        "        song_tensor = tf.convert_to_tensor(song_numpy, dtype=tf.float32)     \n",
        "        songs_to_batch.append(song_tensor)\n",
        "\n",
        "    songs_batch = tf.stack(songs_to_batch, axis=0)             \n",
        "    return songs_batch        "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WOOrgGcOawFn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# create_pivot_batch funcionality:\n",
        "# As input_batch is user_id, track_id coulmns and output_batch is basically rating\n",
        "# So, input batch needs to be converted to X matrix i.e. user vs song rating matrix\n",
        "# Each song is basically timesteps, features and stacked together to create a batch.\n",
        "def create_pivot_batch(input_batch, target_batch):    \n",
        "    # Creating X matrix, user_index: 0 and song_index:1    \n",
        "    rows, row_pos = np.unique(input_batch[:, 0], return_inverse=True)\n",
        "    cols, col_pos = np.unique(input_batch[:, 1], return_inverse=True)\n",
        "    X = np.zeros((BATCH_SIZE, BATCH_SIZE))\n",
        "    X[row_pos, col_pos] = target_batch\n",
        "    return X"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bfe61T08axbX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        },
        "outputId": "c4531039-fdc9-42ab-e8b5-883387a4aef3"
      },
      "source": [
        "# Testing dataset loop, to see how it works.\n",
        "print(\"# (batch_number, input_batch:(batch_size, [user_id, track_id]),\\\n",
        " \\n output_batch:(batch_size, target), songs_batch:(batch_size, timesteps, features))\")\n",
        "\n",
        "for (batch, (input_batch, target_batch)) in enumerate(train_dataset.take(2)):    \n",
        "    songs_batch = create_songs_batch(input_batch)\n",
        "    X = create_pivot_batch(input_batch, target_batch)\n",
        "    user_index = input_batch[:, 0].numpy()\n",
        "    print(batch, input_batch.shape, target_batch.shape, songs_batch.shape, X.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "# (batch_number, input_batch:(batch_size, [user_id, track_id]), \n",
            " output_batch:(batch_size, target), songs_batch:(batch_size, timesteps, features))\n",
            "0 (500, 2) (500,) (500, 2600, 32) (500, 500)\n",
            "1 (500, 2) (500,) (500, 2600, 32) (500, 500)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QxcbtU-Hayj_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class HybridFactorization(tf.keras.layers.Layer):\n",
        "    # embedding_size is also the number of lstm units\n",
        "    # num_users, num_movies = input_shape\n",
        "    # required_users: (batch_size, embedding_size)\n",
        "    # songs_output: (batch_size, embedding_size)\n",
        "    def __init__(self, embedding_size, num_users, num_tracks):        \n",
        "        super(HybridFactorization, self).__init__()\n",
        "        self.embedding_size = embedding_size    \n",
        "        self.num_users = num_users\n",
        "        self.num_tracks = num_tracks  \n",
        "        self.required_users = None         \n",
        "        self.U = self.add_weight(\"U\", \n",
        "                                shape=[self.num_users, self.embedding_size], \n",
        "                                dtype=tf.float32,\n",
        "                                initializer=tf.initializers.GlorotUniform)                        \n",
        "        self.lstm = tf.keras.layers.LSTM(self.embedding_size) \n",
        "\n",
        "    def call(self, user_index, songs_batch):\n",
        "        # output_lstm: (batch_size, emb_sz)\n",
        "        # batch_encoding: (batch_size, num_users)\n",
        "        # required_users: (batch_size, emb_sz)\n",
        "        output_lstm = self.lstm(songs_batch)        \n",
        "\n",
        "        user_idx = np.array(user_index)\n",
        "        batch_encoding = np.zeros((user_idx.size, self.num_users))        \n",
        "        batch_encoding[np.arange(user_idx.size), user_idx] = 1\n",
        "        batch_encoding = tf.convert_to_tensor(batch_encoding, dtype=tf.float32)\n",
        "        \n",
        "        self.required_users = tf.matmul(batch_encoding, self.U)\n",
        "        return tf.matmul(self.required_users, output_lstm, transpose_b=True)                    \n",
        "\n",
        "\n",
        "class HybridRecommender(tf.keras.Model):\n",
        "    def __init__(self, embedding_size, num_users, num_tracks):\n",
        "        super(HybridRecommender, self).__init__()\n",
        "        self.HybridFactorization = HybridFactorization(embedding_size, \n",
        "                                                       num_users, num_tracks)        \n",
        "\n",
        "    def call(self, user_index, songs_batch):\n",
        "        output = self.HybridFactorization(user_index, songs_batch)        \n",
        "        return output  "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gxc-fpj7bIIT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 197
        },
        "outputId": "48f2a3eb-e69d-4917-d024-fae7f5817490"
      },
      "source": [
        "# Building Model\n",
        "model = HybridRecommender(EMBEDDING_SIZE, num_users, num_tracks)\n",
        "Xhat = model(user_index, songs_batch)        \n",
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"hybrid_recommender\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "hybrid_factorization (Hybrid multiple                  1622950   \n",
            "=================================================================\n",
            "Total params: 1,622,950\n",
            "Trainable params: 1,622,950\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9kEvH0s9bNve",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "e39dfdfd-67a9-473d-ad67-9da59e835489"
      },
      "source": [
        "# Add regularization, if needed\n",
        "def loss_fn(source, target):            \n",
        "    mse = tf.keras.losses.MeanSquaredError()        \n",
        "    return mse(source, target)\n",
        "\n",
        "loss_fn(X, Xhat)    "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(), dtype=float32, numpy=0.01287853>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yDBt1K3Vs7OF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "checkpoint_dir = PATH_MODEL + 'training_checkpoints'\n",
        "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n",
        "\n",
        "# Create a callback that saves the model's weights\n",
        "cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_prefix,\n",
        "                                                 save_weights_only=True,\n",
        "                                                 verbose=1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4yZ8gSFV9O0W",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "optimizer = tf.keras.optimizers.Adam()\n",
        "model.compile(optimizer, loss_fn)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p972itH1LAGl",
        "colab_type": "text"
      },
      "source": [
        "## Batch size model implementation."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vxpO0MbjbRC8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# In this approach, Xhat is batch_users vs batch_movies. This approach clearly fails\n",
        "# Why? Here we are minimizing X - Xhat. In this as it is a sparse matrix.\n",
        "# So, algorithm can spread small positive values in all positions in matrix where\n",
        "# there are originally zeroes, to lower mse error.\n",
        "# Thought of Need to write another loss_fn for this approach, where we calculate\n",
        "# for only user_id and song_id where a rating is given, but it didn't works, even after\n",
        "# some epochs as it didn't learn.\n",
        "# So, please refer to below single loop implementation as they did in the original paper.\n",
        "\n",
        "tf.keras.backend.clear_session()\n",
        "\n",
        "optimizer = tf.keras.optimizers.Adam()\n",
        "EPOCHS = 1\n",
        "\n",
        "losses = []\n",
        "\n",
        "for epoch in range(EPOCHS):\n",
        "    start = time.time()\n",
        "    total_loss = 0\n",
        "    \n",
        "    for (batch, (input_batch, target_batch)) in enumerate(train_dataset):            \n",
        "        songs_batch = create_songs_batch(input_batch)\n",
        "        user_index = input_batch[:, 0].numpy()\n",
        "        X = create_pivot_batch(input_batch, target_batch)        \n",
        "\n",
        "        with tf.GradientTape() as tape:\n",
        "            Xhat = model(user_index, songs_batch)\n",
        "            batch_loss = loss_fn(X, Xhat)            \n",
        "\n",
        "        variables = model.trainable_variables\n",
        "        gradients = tape.gradient(batch_loss, variables)\n",
        "        optimizer.apply_gradients(zip(gradients, variables))\n",
        "        \n",
        "        total_loss += batch_loss\n",
        "\n",
        "        if batch % 100 == 0:\n",
        "            print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1,\n",
        "                                                        batch,\n",
        "                                                        batch_loss.numpy()))\n",
        "    # saving (checkpoint) the model every 2 epochs\n",
        "    if (epoch + 1) % 2 == 0:\n",
        "        checkpoint.save(file_prefix = checkpoint_prefix)\n",
        "\n",
        "    loss_value = total_loss.numpy()\n",
        "    losses.append(loss_value)            \n",
        "\n",
        "    print('Epoch {} Loss {:.4f}'.format(epoch + 1,\n",
        "                                        total_loss / steps_per_epoch))\n",
        "    print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zVb2AUjwr52s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def plot_graphs(losses, metric):\n",
        "  plt.plot(losses)\n",
        "  plt.xlabel(\"Epochs\")\n",
        "  plt.ylabel(metric)  \n",
        "  plt.show()\n",
        "\n",
        "plot_graphs(losses, 'loss')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qQ_emU6K1Fvz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "model.load_weights(checkpoint_path)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FJIl1qtR7MG-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 197
        },
        "outputId": "32696f3d-621a-4e1f-c004-f4ed0ad7c17c"
      },
      "source": [
        "# Clearly this approach fails by my analysis.\n",
        "for inp, targ in test_dataset.take(10):\n",
        "    # inp.shape: (2), inp_.shape: (1, 2)\n",
        "    inp_ = tf.expand_dims(inp, 0)    \n",
        "    song = create_songs_batch(inp_)\n",
        "    user_index = inp_[:, 0].numpy()\n",
        "    # ans = model(user_index, song)    \n",
        "    # print(ans, targ)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tf.Tensor([[-0.01670487]], shape=(1, 1), dtype=float32) tf.Tensor(1, shape=(), dtype=int64)\n",
            "tf.Tensor([[0.]], shape=(1, 1), dtype=float32) tf.Tensor(5, shape=(), dtype=int64)\n",
            "tf.Tensor([[2.0715503e-34]], shape=(1, 1), dtype=float32) tf.Tensor(5, shape=(), dtype=int64)\n",
            "tf.Tensor([[0.]], shape=(1, 1), dtype=float32) tf.Tensor(5, shape=(), dtype=int64)\n",
            "tf.Tensor([[0.00633507]], shape=(1, 1), dtype=float32) tf.Tensor(1, shape=(), dtype=int64)\n",
            "tf.Tensor([[0.]], shape=(1, 1), dtype=float32) tf.Tensor(2, shape=(), dtype=int64)\n",
            "tf.Tensor([[0.]], shape=(1, 1), dtype=float32) tf.Tensor(2, shape=(), dtype=int64)\n",
            "tf.Tensor([[-0.0312532]], shape=(1, 1), dtype=float32) tf.Tensor(2, shape=(), dtype=int64)\n",
            "tf.Tensor([[0.]], shape=(1, 1), dtype=float32) tf.Tensor(2, shape=(), dtype=int64)\n",
            "tf.Tensor([[0.00495162]], shape=(1, 1), dtype=float32) tf.Tensor(4, shape=(), dtype=int64)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LekH0gS6IdML",
        "colab_type": "text"
      },
      "source": [
        "## Creating a single loop model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XfR5MvqNJfqb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Original Paper implementation\n",
        "# GLOBAL VARIABLES\n",
        "# Here buffer size is whole dataset.\n",
        "BUFFER_SIZE = len(train_df)\n",
        "BATCH_SIZE = 500\n",
        "EMBEDDING_SIZE = 50\n",
        "EPOCHS = 10\n",
        "steps_per_epoch = len(train_df)//BATCH_SIZE"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RpwB9nCDHOtc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_df = df.sample(frac=0.8, random_state=0)\n",
        "test_df = df.drop(train_df.index)\n",
        "\n",
        "train_rating = train_df.pop('rating')\n",
        "test_rating = test_df.pop('rating')\n",
        "\n",
        "train_dataset = tf.data.Dataset.from_tensor_slices((train_df.values, train_rating.values))\n",
        "test_dataset = tf.data.Dataset.from_tensor_slices((test_df.values, test_rating.values))\n",
        "\n",
        "train_dataset = train_dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lBW4kZxmI0_E",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Here we take each user, song entry of a batch and try to figure out loss for each\n",
        "# rating. Takes a lot of time for single epoch. Cannot afford to complete now due to\n",
        "# computation costs.\n",
        "\n",
        "tf.keras.backend.clear_session()\n",
        "\n",
        "optimizer = tf.keras.optimizers.Adam()\n",
        "EPOCHS = 1\n",
        "\n",
        "losses = []\n",
        "\n",
        "for epoch in range(EPOCHS):\n",
        "    start = time.time()\n",
        "    total_loss = 0\n",
        "    \n",
        "    for (batch, (input_batch, target_batch)) in enumerate(train_dataset):                \n",
        "        target_batch = tf.dtypes.cast(target_batch, tf.float32)\n",
        "        # model_predictions = []\n",
        "\n",
        "        batch_loss = 0\n",
        "        for inp, tar in zip(input_batch, target_batch):            \n",
        "            # inp.shape: (2), inp_.shape: (1, 2)            \n",
        "            inp_ = tf.expand_dims(inp, 0)        \n",
        "            song = create_songs_batch(inp_)\n",
        "            user_index = inp_[:, 0].numpy()\n",
        "            \n",
        "            with tf.GradientTape() as tape:            \n",
        "                pred = model(user_index, song)                \n",
        "                batch_loss += loss_fn(tar, pred)    \n",
        "        \n",
        "            # model_predictions: list (batch_size, 1, 1)\n",
        "            # model_predictions.append(model(user_index, song))            \n",
        "\n",
        "        # preds = np.squeeze(model_predictions)\n",
        "        # preds = tf.convert_to_tensor(preds, dtype=tf.float32)                 \n",
        "        \n",
        "        batch_loss = batch_loss//input_batch.shape[0]\n",
        "        variables = model.trainable_variables\n",
        "        gradients = tape.gradient(batch_loss, variables)\n",
        "        optimizer.apply_gradients(zip(gradients, variables))\n",
        "        \n",
        "        total_loss += batch_loss\n",
        "\n",
        "        if batch % 100 == 0:\n",
        "            print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1,\n",
        "                                                        batch,\n",
        "                                                        batch_loss.numpy()))\n",
        "    # saving (checkpoint) the model every 2 epochs\n",
        "    if (epoch + 1) % 2 == 0:\n",
        "        checkpoint.save(file_prefix = checkpoint_prefix)\n",
        "\n",
        "    loss_value = total_loss.numpy()\n",
        "    losses.append(loss_value)            \n",
        "\n",
        "    print('Epoch {} Loss {:.4f}'.format(epoch + 1,\n",
        "                                        total_loss / steps_per_epoch))\n",
        "    print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "npi0H8fcV7lZ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "8881c1a6-b5de-42da-8225-0ed0fb370090"
      },
      "source": [
        "# Redundant Code\n",
        "\n",
        "# for (batch, (input_batch, target_batch)) in enumerate(train_dataset.take(1)):                \n",
        "#     target_batch = tf.dtypes.cast(target_batch, tf.float32)\n",
        "#     model_predictions = []\n",
        "#     for inp in input_batch:\n",
        "#         # inp.shape: (2), inp_.shape: (1, 2)\n",
        "#         # model_predictions: list (batch_size, 1, 1)\n",
        "#         inp_ = tf.expand_dims(inp, 0)        \n",
        "#         song = create_songs_batch(inp_)\n",
        "#         user_index = inp_[:, 0].numpy()\n",
        "#         model_predictions.append(model(user_index, song))            \n",
        "\n",
        "#     preds = np.squeeze(model_predictions)\n",
        "#     preds = tf.convert_to_tensor(preds, dtype=tf.float32)                 \n",
        "\n",
        "# loss_fn(target_batch, preds)\n",
        "\n",
        "# aa = tf.reshape(ans, [])\n",
        "# tt = tf.dtypes.cast(tar, tf.float32)\n",
        "# los = (tt - ans).numpy()\n",
        "# los += (tt - ans).numpy()\n",
        "\n",
        "# lss = []\n",
        "# lss.append(ans)\n",
        "# xx = np.array(lss)\n",
        "# asas = np.squeeze(xx)\n",
        "# xxx = tf.convert_to_tensor(asas, dtype=tf.float32)     "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tf.Tensor: shape=(), dtype=float32, numpy=5.9311852>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 91
        }
      ]
    }
  ]
}