data_preproc.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data preprocessing\n",
    "<a id='#Data_preprocessing'></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MIND-small:\n",
      "Dataset         Train shape          Test shape\n",
      "--------------------------------------------------\n",
      "news            (51282, 8)           (42416, 8)\n",
      "behavior        (156965, 5)          (73152, 5)\n",
      "entity          (26904, 102)         (22893, 102)\n",
      "relation        (1091, 102)          (1091, 102)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Impression ID</th>\n",
       "      <th>User ID</th>\n",
       "      <th>Time</th>\n",
       "      <th>History</th>\n",
       "      <th>Impressions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>U13740</td>\n",
       "      <td>11/11/2019 9:05:58 AM</td>\n",
       "      <td>N55189 N42782 N34694 N45794 N18445 N63302 N104...</td>\n",
       "      <td>N55689-1 N35729-0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Impression ID User ID                   Time  \\\n",
       "0              1  U13740  11/11/2019 9:05:58 AM   \n",
       "\n",
       "                                             History        Impressions  \n",
       "0  N55189 N42782 N34694 N45794 N18445 N63302 N104...  N55689-1 N35729-0  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "news_train = 'data/MINDsmall_train/news.tsv'\n",
    "behavior_train = 'data/MINDsmall_train/behaviors.tsv'\n",
    "entity_train = 'data/MINDsmall_train/entity_embedding.vec'\n",
    "relation_train = 'data/MINDsmall_train/relation_embedding.vec'\n",
    "#-------------------------------------------\n",
    "news_test = 'data/MINDsmall_dev/news.tsv'\n",
    "behavior_test = 'data/MINDsmall_dev/behaviors.tsv'\n",
    "entity_test = 'data/MINDsmall_dev/entity_embedding.vec'\n",
    "relation_test = 'data/MINDsmall_dev/relation_embedding.vec' \n",
    "\n",
    "def load_df(path):\n",
    "    if 'news' in path:\n",
    "        columns = ['News ID',\n",
    "                \"Category\",\n",
    "                \"SubCategory\",\n",
    "                \"Title\",\n",
    "                \"Abstract\",\n",
    "                \"URL\",\n",
    "                \"Title Entities\",\n",
    "                \"Abstract Entities\"]\n",
    "    \n",
    "    elif 'behavior' in path:\n",
    "        columns = ['Impression ID',\n",
    "                \"User ID\",\n",
    "                \"Time\",\n",
    "                \"History\",\n",
    "                \"Impressions\"]\n",
    "    else:\n",
    "        return pd.read_csv(path, sep='\\t', header=None)\n",
    "    \n",
    "    df = pd.read_csv(path, sep='\\t', header=None, names=columns)\n",
    "    return df\n",
    "\n",
    "news_train, news_test, behavior_train, behavior_test = map(load_df, [news_train, news_test, behavior_train, behavior_test])\n",
    "entity_train, relation_train, entity_test, relation_test = map(load_df, [entity_train, relation_train, entity_test, relation_test])\n",
    "print('MIND-small:')\n",
    "print(f\"{'Dataset':<15} {'Train shape':<20} {'Test shape'}\")\n",
    "print(f\"{'-'*50}\")\n",
    "print(f\"{'news':<15} {str(news_train.shape):<20} {news_test.shape}\")\n",
    "print(f\"{'behavior':<15} {str(behavior_train.shape):<20} {behavior_test.shape}\")\n",
    "print(f\"{'entity':<15} {str(entity_train.shape):<20} {entity_test.shape}\")\n",
    "print(f\"{'relation':<15} {str(relation_train.shape):<20} {relation_test.shape}\")\n",
    "behavior_train.head(1)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "train data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5110877, 3)\n",
      "(915903, 3)\n"
     ]
    }
   ],
   "source": [
    "#prepare history data\n",
    "behavior_train[\"History\"] = behavior_train[\"History\"].apply(lambda x: x.split(\" \") if isinstance(x, str) else []) #make it iterable\n",
    "train = behavior_train.drop([\"Impression ID\", \"Time\", \"Impressions\"], axis=1)\n",
    "train = train.explode(\"History\")\n",
    "train['label'] = 1\n",
    "train.rename(columns={ \"User ID\": \"user_id:token\" ,\"History\": \"item_id:token\", \"label\": \"label:float\"}, inplace=True)\n",
    "train['user_id:token'] = train['user_id:token'].str[1:]\n",
    "train['item_id:token'] = train['item_id:token'].str[1:]\n",
    "print(train.shape)\n",
    "train.drop_duplicates(inplace=True)\n",
    "print(train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5843444, 3)\n",
      "(5033875, 3)\n"
     ]
    }
   ],
   "source": [
    "#same with impressions\n",
    "behavior_train[\"Impressions\"] = behavior_train[\"Impressions\"].apply(lambda x: x.split(\" \") if isinstance(x, str) else [])#make it iterable\n",
    "imp = behavior_train.drop([\"Impression ID\", \"Time\", \"History\"], axis=1)\n",
    "imp = imp.explode(\"Impressions\")\n",
    "imp['label'] = imp['Impressions'].apply(lambda x: int(x.split(\"-\")[-1]))\n",
    "#delete last two character of impressions\n",
    "imp['Impressions'] = imp['Impressions'].apply(lambda x: x[:-2])\n",
    "imp.rename(columns={\"User ID\": \"user_id:token\" ,\"Impressions\": \"item_id:token\", \"label\": \"label:float\"}, inplace=True)\n",
    "imp['user_id:token'] = imp['user_id:token'].str[1:]\n",
    "imp['item_id:token'] = imp['item_id:token'].str[1:]\n",
    "print(imp.shape)\n",
    "imp.drop_duplicates(inplace=True)\n",
    "print(imp.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5949778, 3)\n",
      "(5948746, 3)\n"
     ]
    }
   ],
   "source": [
    "inter_train = pd.concat([train, imp], axis=0)\n",
    "print(inter_train.shape)\n",
    "inter_train.drop_duplicates(inplace=True)\n",
    "print(inter_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def five_core_filter(df):\n",
    "    while True:\n",
    "        before_shape = df.shape\n",
    "        #users with at least 5 tracks\n",
    "        counts = df['user_id:token'].value_counts()\n",
    "        df = df[df['user_id:token'].isin(counts[counts >= 5].index)]\n",
    "        #tracks with at least 5 users\n",
    "        counts = df['item_id:token'].value_counts()\n",
    "        df = df[df['item_id:token'].isin(counts[counts >= 5].index)]\n",
    "        \n",
    "        after_shape = df.shape\n",
    "        #stop condition\n",
    "        if before_shape == after_shape:\n",
    "            break\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before filter: (5948746, 3) unique users: 50000 unique items: 51282\n",
      "After filter: (5900343, 3) unique users: 49886 unique items: 24011\n"
     ]
    }
   ],
   "source": [
    "print('Before filter:', inter_train.shape, \"unique users:\", inter_train['user_id:token'].nunique(), \"unique items:\", inter_train['item_id:token'].nunique())\n",
    "inter_train = five_core_filter(inter_train)\n",
    "print('After filter:', inter_train.shape, \"unique users:\", inter_train['user_id:token'].nunique(), \"unique items:\", inter_train['item_id:token'].nunique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id:token</th>\n",
       "      <th>item_id:token</th>\n",
       "      <th>label:float</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13740</td>\n",
       "      <td>55189</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13740</td>\n",
       "      <td>42782</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13740</td>\n",
       "      <td>34694</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  user_id:token item_id:token  label:float\n",
       "0         13740         55189            1\n",
       "0         13740         42782            1\n",
       "0         13740         34694            1"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#save\n",
    "inter_train.to_csv('mind_train.inter', index=False, sep='\\t')\n",
    "inter_train.head(3)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "------------------------------------------------------------------------------------------\n",
    "\n",
    "TEST DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2364728, 3)\n",
      "(1260987, 3)\n"
     ]
    }
   ],
   "source": [
    "#prepare history data\n",
    "behavior_test[\"History\"] = behavior_test[\"History\"].apply(lambda x: x.split(\" \") if isinstance(x, str) else []) #make it iterable\n",
    "test = behavior_test.drop([\"Impression ID\", \"Time\", \"Impressions\"], axis=1)\n",
    "test = test.explode(\"History\")\n",
    "test['label'] = 1\n",
    "test.rename(columns={ \"User ID\": \"user_id:token\" ,\"History\": \"item_id:token\", \"label\": \"label:float\"}, inplace=True)\n",
    "test['user_id:token'] = test['user_id:token'].str[1:]\n",
    "test['item_id:token'] = test['item_id:token'].str[1:]\n",
    "print(test.shape)\n",
    "test.drop_duplicates(inplace=True)\n",
    "print(test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2740998, 3)\n",
      "(2478564, 3)\n"
     ]
    }
   ],
   "source": [
    "#same with impressions\n",
    "behavior_test[\"Impressions\"] = behavior_test[\"Impressions\"].apply(lambda x: x.split(\" \") if isinstance(x, str) else [])#make it iterable\n",
    "imp = behavior_test.drop([\"Impression ID\", \"Time\", \"History\"], axis=1)\n",
    "imp = imp.explode(\"Impressions\")\n",
    "imp['label'] = imp['Impressions'].apply(lambda x: int(x.split(\"-\")[-1]))\n",
    "#delete last two character of impressions\n",
    "imp['Impressions'] = imp['Impressions'].apply(lambda x: x[:-2])\n",
    "imp.rename(columns={\"User ID\": \"user_id:token\" ,\"Impressions\": \"item_id:token\", \"label\": \"label:float\"}, inplace=True)\n",
    "imp['user_id:token'] = imp['user_id:token'].str[1:]\n",
    "imp['item_id:token'] = imp['item_id:token'].str[1:]\n",
    "print(imp.shape)\n",
    "imp.drop_duplicates(inplace=True)\n",
    "print(imp.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3739551, 3)\n",
      "(3739366, 3)\n"
     ]
    }
   ],
   "source": [
    "inter_test = pd.concat([test, imp], axis=0)\n",
    "print(inter_test.shape)\n",
    "inter_test.drop_duplicates(inplace=True)\n",
    "print(inter_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(50000, 42416)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#remove from test set users and items that are not in train set\n",
    "#inter_test = inter_test[inter_test['user_id:token'].isin(inter_train['user_id:token'])]\n",
    "#inter_test = inter_test[inter_test['item_id:token'].isin(inter_train['item_id:token'])]\n",
    "#print n unique users and items\n",
    "inter_test['user_id:token'].nunique(), inter_test['item_id:token'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "#save\n",
    "inter_test.to_csv('mind_test.inter', index=False, sep='\\t')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "------------------------------------------------------------------------------------------\n",
    "FULL DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(9639709, 3)\n",
      "(9471416, 3)\n"
     ]
    }
   ],
   "source": [
    "inter = pd.concat([inter_train, inter_test], axis=0)\n",
    "print(inter.shape)\n",
    "inter.drop_duplicates(inplace=True)\n",
    "print(inter.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique users: 93960 Unique items: 51218\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id:token</th>\n",
       "      <th>item_id:token</th>\n",
       "      <th>label:float</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13740</td>\n",
       "      <td>55189</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13740</td>\n",
       "      <td>42782</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13740</td>\n",
       "      <td>34694</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  user_id:token item_id:token  label:float\n",
       "0         13740         55189            1\n",
       "0         13740         42782            1\n",
       "0         13740         34694            1"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#get unique users and items\n",
    "print('Unique users:', inter['user_id:token'].nunique(), 'Unique items:', inter['item_id:token'].nunique())\n",
    "inter.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "label\n",
       "0    7140340\n",
       "1    2329669\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#save\n",
    "#print(inter.isnull().sum())\n",
    "#delete rows with invalid id\n",
    "#inter = inter.dropna(subset=['user_id', 'item_id'])\n",
    "#print(inter.isnull().sum())\n",
    "#make all columns integer\n",
    "inter = inter.astype(int)\n",
    "inter.to_csv('data/mind.inter', index=False, sep='\\t')\n",
    "inter['label'].value_counts()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "–––––––––"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load df\n",
    "#import pandas as pd\n",
    "\n",
    "#inter = pd.read_csv('mind_small/mind_small.inter', sep='\\t')\n",
    "#inter.shape"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data splitting <br>\n",
    "<a id='#Datasplitting'></a>\n",
    "for each user, their interaxtion are splitted into train, test, and validation data (<i>avoid cold start for now as it is not the focus of my research</i>)<br>\n",
    "ration: 0.6, 0.2, and 0.2 for train, test, and validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "def five_core_filter(df, n):\n",
    "    while True:\n",
    "        before_shape = df.shape\n",
    "        #users with at least 5 tracks\n",
    "        counts = df['user_id'].value_counts()\n",
    "        df = df[df['user_id'].isin(counts[counts >= n].index)]\n",
    "        #tracks with at least 5 users\n",
    "        counts = df['item_id'].value_counts()\n",
    "        df = df[df['item_id'].isin(counts[counts >= n].index)]\n",
    "        \n",
    "        after_shape = df.shape\n",
    "        #stop condition\n",
    "        if before_shape == after_shape:\n",
    "            break\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(9470009, 3)\n",
      "(9432207, 3)\n"
     ]
    }
   ],
   "source": [
    "data = pd.read_csv('data/mind.inter', sep='\\t')\n",
    "print(data.shape)\n",
    "new_data = five_core_filter(data, 5)\n",
    "print(new_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(5622131, 3) (1886241, 3) (1923835, 3)\n"
     ]
    }
   ],
   "source": [
    "def custom_train_test_val_split(data, train_size, test_size, val_size):\n",
    "    train = pd.DataFrame()\n",
    "    test = pd.DataFrame()\n",
    "    val = pd.DataFrame()\n",
    "    #for every user\n",
    "    grouped = data.groupby('user_id')\n",
    "    #split the data\n",
    "    for _, group in grouped:\n",
    "        #print(_)\n",
    "        if len(group) < 5:  #if a user has <5 interaction\n",
    "                            #I f* up the preprocessing\n",
    "            train = pd.concat([train, group])\n",
    "            print('ouch this should have never been printed :(')\n",
    "        else:\n",
    "            train_group, temp_group = train_test_split(group, train_size=train_size, random_state=42)\n",
    "            if len(temp_group) > 1:\n",
    "                test_group, val_group = train_test_split(temp_group, test_size=val_size/(test_size + val_size), random_state=42)\n",
    "                test = pd.concat([test, test_group])\n",
    "                val = pd.concat([val, val_group])\n",
    "            else:\n",
    "                if test_size > val_size:\n",
    "                    test = pd.concat([test, temp_group])\n",
    "                else:\n",
    "                    val = pd.concat([val, temp_group])\n",
    "                \n",
    "            train = pd.concat([train, train_group])\n",
    "\n",
    "    return train, test, val\n",
    "\n",
    "\n",
    "train, test, val = custom_train_test_val_split(new_data, train_size=0.6, test_size=0.2, val_size=0.2)\n",
    "print(train.shape, test.shape, val.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train: 93734 Test: 93734 Val: 93734\n",
      "Train: 27941 Test: 26247 Val: 26377\n",
      "Label distribution:\n",
      " label\n",
      "0    0.765671\n",
      "1    0.234329\n",
      "Name: proportion, dtype: float64 label\n",
      "0    0.744905\n",
      "1    0.255095\n",
      "Name: proportion, dtype: float64 label\n",
      "0    0.742487\n",
      "1    0.257513\n",
      "Name: proportion, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "print('Train:', train['user_id'].nunique(), 'Test:', test['user_id'].nunique(), 'Val:', val['user_id'].nunique())\n",
    "print('Train:', train['item_id'].nunique(), 'Test:', test['item_id'].nunique(), 'Val:', val['item_id'].nunique())\n",
    "print('Label distribution:\\n',  train['label'].value_counts(normalize=True), test['label'].value_counts(normalize=True), val['label'].value_counts(normalize=True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "#save\n",
    "train.to_csv('data/train', index=False, sep='\\t')\n",
    "test.to_csv('data/test', index=False, sep='\\t')\n",
    "val.to_csv('data/val', index=False, sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train: 0 Test: 0 Val: 0\n",
      "Train: 0 Test: 0 Val: 0\n"
     ]
    }
   ],
   "source": [
    "#check for duplicates\n",
    "print('Train:', train.duplicated().sum(), 'Test:', test.duplicated().sum(), 'Val:', val.duplicated().sum())\n",
    "#check if same interactions are in train and test   \n",
    "print('Train:', train.isin(test).sum().sum(), 'Test:', test.isin(train).sum().sum(), 'Val:', val.isin(train).sum().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exact Row Overlap: Train-Test: 0 Train-Val: 0 Test-Val: 0\n"
     ]
    }
   ],
   "source": [
    "print('Exact Row Overlap: Train-Test:', pd.merge(train, test, on=['user_id', 'item_id', 'label'], how='inner').shape[0],\n",
    "      'Train-Val:', pd.merge(train, val, on=['user_id', 'item_id', 'label'], how='inner').shape[0],\n",
    "      'Test-Val:', pd.merge(test, val, on=['user_id', 'item_id', 'label'], how='inner').shape[0])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "recbole-env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.19"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}