P2 - Item To Item.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "083784c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "61d07f88",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userID</th>\n",
       "      <th>movieID</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4490</td>\n",
       "      <td>2109</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5839</td>\n",
       "      <td>3471</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5382</td>\n",
       "      <td>150</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1262</td>\n",
       "      <td>1237</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6005</td>\n",
       "      <td>2273</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   userID  movieID  rating\n",
       "0    4490     2109       4\n",
       "1    5839     3471       4\n",
       "2    5382      150       3\n",
       "3    1262     1237       5\n",
       "4    6005     2273       4"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_data = pd.read_csv(\"train.csv\")\n",
    "raw_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "781c7f0a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>userID</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>...</th>\n",
       "      <th>6031</th>\n",
       "      <th>6032</th>\n",
       "      <th>6033</th>\n",
       "      <th>6034</th>\n",
       "      <th>6035</th>\n",
       "      <th>6036</th>\n",
       "      <th>6037</th>\n",
       "      <th>6038</th>\n",
       "      <th>6039</th>\n",
       "      <th>6040</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>movieID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 6040 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "userID   1     2     3     4     5     6     7     8     9     10    ...   \n",
       "movieID                                                              ...   \n",
       "1         3.0   NaN   NaN   NaN   NaN   4.0   NaN   NaN   4.0   NaN  ...  \\\n",
       "2         NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "3         NaN   NaN   NaN   NaN   NaN   1.0   NaN   NaN   NaN   NaN  ...   \n",
       "4         NaN   NaN   NaN   NaN   NaN   2.0   NaN   NaN   NaN   NaN  ...   \n",
       "5         NaN   NaN   NaN   NaN   NaN   1.0   NaN   NaN   NaN   NaN  ...   \n",
       "\n",
       "userID   6031  6032  6033  6034  6035  6036  6037  6038  6039  6040  \n",
       "movieID                                                              \n",
       "1         5.0   5.0   4.0   NaN   4.0   NaN   NaN   NaN   NaN   5.0  \n",
       "2         5.0   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "3         NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "4         NaN   NaN   3.0   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "5         NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "\n",
       "[5 rows x 6040 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pivot_ii= raw_data.pivot(index=\"movieID\", columns= \"userID\", values=\"rating\")\n",
    "pivot_ii.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1a157e14",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_mean = pivot_ii.mean().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ef7648a3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3705, 6040)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pivot_ii.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e37408fe",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>userID</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>...</th>\n",
       "      <th>6031</th>\n",
       "      <th>6032</th>\n",
       "      <th>6033</th>\n",
       "      <th>6034</th>\n",
       "      <th>6035</th>\n",
       "      <th>6036</th>\n",
       "      <th>6037</th>\n",
       "      <th>6038</th>\n",
       "      <th>6039</th>\n",
       "      <th>6040</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>movieID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1.145287</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.145287</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.145287</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.854713</td>\n",
       "      <td>0.854713</td>\n",
       "      <td>-0.145287</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.145287</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.854713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.800000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-2.021142</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.733728</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.266272</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-2.017007</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 6040 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "userID       1     2     3     4     5         6     7     8         9      \n",
       "movieID                                                                     \n",
       "1       -1.145287   0.0   0.0   0.0   0.0 -0.145287   0.0   0.0 -0.145287  \\\n",
       "2        0.000000   0.0   0.0   0.0   0.0  0.000000   0.0   0.0  0.000000   \n",
       "3        0.000000   0.0   0.0   0.0   0.0 -2.021142   0.0   0.0  0.000000   \n",
       "4        0.000000   0.0   0.0   0.0   0.0 -0.733728   0.0   0.0  0.000000   \n",
       "5        0.000000   0.0   0.0   0.0   0.0 -2.017007   0.0   0.0  0.000000   \n",
       "\n",
       "userID   10    ...      6031      6032      6033  6034      6035  6036  6037   \n",
       "movieID        ...                                                             \n",
       "1         0.0  ...  0.854713  0.854713 -0.145287   0.0 -0.145287   0.0   0.0  \\\n",
       "2         0.0  ...  1.800000  0.000000  0.000000   0.0  0.000000   0.0   0.0   \n",
       "3         0.0  ...  0.000000  0.000000  0.000000   0.0  0.000000   0.0   0.0   \n",
       "4         0.0  ...  0.000000  0.000000  0.266272   0.0  0.000000   0.0   0.0   \n",
       "5         0.0  ...  0.000000  0.000000  0.000000   0.0  0.000000   0.0   0.0   \n",
       "\n",
       "userID   6038  6039      6040  \n",
       "movieID                        \n",
       "1         0.0   0.0  0.854713  \n",
       "2         0.0   0.0  0.000000  \n",
       "3         0.0   0.0  0.000000  \n",
       "4         0.0   0.0  0.000000  \n",
       "5         0.0   0.0  0.000000  \n",
       "\n",
       "[5 rows x 6040 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scaled_df = pivot_ii - pivot_ii.mean(axis=1).values.reshape(-1, 1)\n",
    "scaled_df.fillna(0, inplace=True)\n",
    "scaled_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b39e4dbf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>IDs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>629_2683</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>601_904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5673_3717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5616_1721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1099_3697</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         IDs\n",
       "0   629_2683\n",
       "1    601_904\n",
       "2  5673_3717\n",
       "3  5616_1721\n",
       "4  1099_3697"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_no_labels = pd.read_csv(\"test_without_labels.csv\")\n",
    "test_no_labels.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "50697be4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>IDs</th>\n",
       "      <th>userID</th>\n",
       "      <th>movieID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>629_2683</td>\n",
       "      <td>629</td>\n",
       "      <td>2683</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>601_904</td>\n",
       "      <td>601</td>\n",
       "      <td>904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5673_3717</td>\n",
       "      <td>5673</td>\n",
       "      <td>3717</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5616_1721</td>\n",
       "      <td>5616</td>\n",
       "      <td>1721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1099_3697</td>\n",
       "      <td>1099</td>\n",
       "      <td>3697</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         IDs  userID  movieID\n",
       "0   629_2683     629     2683\n",
       "1    601_904     601      904\n",
       "2  5673_3717    5673     3717\n",
       "3  5616_1721    5616     1721\n",
       "4  1099_3697    1099     3697"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_no_labels[\"userID\"] = test_no_labels.IDs.str.split('_').str[0].astype(int)\n",
    "test_no_labels[\"movieID\"] = test_no_labels.IDs.str.split('_').str[1].astype(int)\n",
    "test_no_labels.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a648d50f",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ii = test_no_labels.drop(\"IDs\",axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5e89c781",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_ratings_ii = []\n",
    "\n",
    "for userId,movie in test_ii.itertuples(index=False):\n",
    "    k = 50\n",
    "    rated_indices = pivot_ii[np.isnan(pivot_ii[userId]) == False].index\n",
    "    filtered_df = scaled_df.loc[rated_indices, :]\n",
    "    if len(filtered_df) == 0:\n",
    "        pred = all_mean\n",
    "    else:\n",
    "        if len(filtered_df) < k:\n",
    "            k = len(filtered_df)\n",
    "        \n",
    "        nn = NearestNeighbors(n_neighbors=k, algorithm=\"kd_tree\")\n",
    "        nn.fit(filtered_df)\n",
    "        test_movie = scaled_df.loc[[movie], :]\n",
    "        close_indices = nn.kneighbors(test_movie ,return_distance=False)[0]\n",
    "        close_friends = filtered_df.iloc[close_indices, :]\n",
    "        pred = pivot_ii.loc[movie, :].mean() + close_friends[userId].mean()\n",
    "        pred_ratings_ii.append(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "952630ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9999"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pred_ratings_ii)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3a6c867c",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(pred_ratings_ii)): \n",
    "    if pred_ratings_ii[i] > 5:\n",
    "        pred_ratings_ii[i] = 5\n",
    "    elif pred_ratings_ii[i] < 1:\n",
    "        pred_ratings_ii[i] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "27a17ac9",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_no_labels[\"rating\"] = pred_ratings_ii\n",
    "test_no_labels.drop(columns=[\"userID\",\"movieID\"],axis=1,inplace=True)\n",
    "test_no_labels.to_csv(\"result_ii_2804.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd529e24",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}