1b_prep_ltr_knn_search.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "import joblib as jb\n",
    "\n",
    "import json\n",
    "import tqdm\n",
    "import joblib\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_parquet(\"./data/train.parquet\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = pd.read_parquet(\"./data/test.parquet\")\n",
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "item_data = pd.read_parquet(\"./data/item_data.parquet\")\n",
    "item_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "item_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "item_title_map = item_data[['item_id', 'title']].drop_duplicates()\n",
    "item_title_map = item_title_map.set_index(\"item_id\").squeeze().to_dict()\n",
    "\n",
    "item_price_map = item_data[['item_id', 'price']].drop_duplicates()\n",
    "item_price_map = item_price_map.set_index(\"item_id\").squeeze().to_dict()\n",
    "\n",
    "item_domain_map = item_data[['item_id', 'domain_id']].drop_duplicates()\n",
    "item_domain_map = item_domain_map.set_index(\"item_id\").squeeze().to_dict()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# knn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "import nmslib\n",
    "index = nmslib.init()\n",
    "index.loadIndex('22a_sbert_neuralmind.nms')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embs_np = joblib.load(\"22a_embs_np.pkl.z\")\n",
    "item_emb_map = {t: embs_np[i] for i, t in enumerate(item_data['item_id'].values)} "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embs_search_np = joblib.load(\"22a_embs_search_np.pkl.z\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "k=50"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "data = []\n",
    "seq_index = 0\n",
    "for hist, bought in tqdm.tqdm(train[['user_history', 'item_bought']].values):\n",
    "\n",
    "    \n",
    "    recall = False\n",
    "    last_ts = None\n",
    "    seq = 0\n",
    "    ts = 0\n",
    "    rep = dict()\n",
    "    for item in json.loads(hist):\n",
    "        i = item['event_info']\n",
    "        item['bought_id'] = bought\n",
    "        item['bought_title'] = item_title_map[bought]\n",
    "        item['bought_price'] = item_price_map[bought]\n",
    "        item['bought_domain'] = item_domain_map[bought]\n",
    "        \n",
    "        if item['event_type'] == 'view':\n",
    "            item['item_title'] = item_title_map[i]\n",
    "            item['item_price'] = item_price_map[i]\n",
    "            item['item_domain'] = item_domain_map[i]\n",
    "            item['has_bought'] = int(bought == i)\n",
    "            item['pt'] = int('MLB' in item['item_domain']) if item['item_domain'] else np.nan\n",
    "            item['viewed'] = 1\n",
    "            rep[i] = item_emb_map[i]\n",
    "        \n",
    "        \n",
    "        \n",
    "        item['seq_pos'] = seq\n",
    "        item['seq_index'] = seq_index\n",
    "\n",
    "        seq += 1\n",
    "        data.append(item)\n",
    "    \n",
    "    lrep = list(rep.values())\n",
    "    if len(lrep) == 0:\n",
    "        view_embedding_mean = embs_search_np[seq_index, :] #search para quem nao tem views\n",
    "    else:\n",
    "        view_embedding_mean = np.mean(lrep, axis=0)\n",
    "    for neighbor in index.knnQuery(view_embedding_mean, k=k)[0]:\n",
    "        item = dict()\n",
    "        i = neighbor\n",
    "        item['event_info'] = neighbor\n",
    "        item['event_type'] = 'knn'\n",
    "        item['bought_id'] = bought\n",
    "        item['bought_title'] = item_title_map[bought]\n",
    "        item['bought_price'] = item_price_map[bought]\n",
    "        item['bought_domain'] = item_domain_map[bought]\n",
    "        item['item_title'] = item_title_map[i]\n",
    "        item['item_price'] = item_price_map[i]\n",
    "        item['item_domain'] = item_domain_map[i]\n",
    "        item['has_bought'] = int(bought == i)\n",
    "        item['pt'] = int('MLB' in item['item_domain']) if item['item_domain'] else np.nan\n",
    "        item['seq_pos'] = -1\n",
    "        item['seq_index'] = seq_index\n",
    "        item['viewed'] = 0\n",
    "        \n",
    "        data.append(item)\n",
    "        \n",
    "        \n",
    "    seq_index += 1\n",
    "        \n",
    "df = pd.DataFrame(data)\n",
    "del data, embs_search_np\n",
    "gc.collect()\n",
    "df['event_timestamp'] = pd.to_datetime(df['event_timestamp']).dt.tz_localize(None)\n",
    "df[df['event_type'] != 'search'].to_parquet(\"./data/22_train_view_melted.parquet\",engine='fastparquet', compression=None)\n",
    "df[df['event_type'] == 'search'].to_parquet(\"./data/22_train_search_melted.parquet\",engine='fastparquet', compression=None)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embs_search_np = joblib.load(\"22a_embs_search_test_np.pkl.z\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# last k item matches bought item\n",
    "data = []\n",
    "seq_index = 0\n",
    "for hist in tqdm.tqdm(test['user_history'].values):\n",
    "\n",
    "    \n",
    "    last_ts = None\n",
    "    seq = 0\n",
    "    ts = 0\n",
    "    rep = dict()\n",
    "    for item in json.loads(hist):\n",
    "        i = item['event_info']\n",
    "        \n",
    "        if item['event_type'] == 'view':\n",
    "            item['item_title'] = item_title_map[i]\n",
    "            item['item_price'] = item_price_map[i]\n",
    "            item['item_domain'] = item_domain_map[i]\n",
    "            item['pt'] = int('MLB' in item['item_domain']) if item['item_domain'] else np.nan\n",
    "            item['viewed'] = 1\n",
    "            rep[i] = item_emb_map[i]\n",
    "        \n",
    "        item['seq_pos'] = seq\n",
    "        item['seq_index'] = seq_index\n",
    "\n",
    "        seq += 1\n",
    "        data.append(item)\n",
    "        \n",
    "    lrep = list(rep.values())\n",
    "    if len(lrep) == 0:\n",
    "        view_embedding_mean = embs_search_np[seq_index, :]\n",
    "    else:\n",
    "        view_embedding_mean = np.mean(lrep, axis=0)\n",
    "    for neighbor in index.knnQuery(view_embedding_mean, k=k)[0]:\n",
    "        item = dict()\n",
    "        i = neighbor\n",
    "        item['event_info'] = neighbor\n",
    "        item['event_type'] = 'knn'\n",
    "        item['item_title'] = item_title_map[i]\n",
    "        item['item_price'] = item_price_map[i]\n",
    "        item['item_domain'] = item_domain_map[i]\n",
    "        item['pt'] = int('MLB' in item['item_domain']) if item['item_domain'] else np.nan\n",
    "        item['seq_pos'] = -1\n",
    "        item['seq_index'] = seq_index\n",
    "        item['viewed'] = 0\n",
    "        \n",
    "        data.append(item)\n",
    "        \n",
    "    seq_index += 1\n",
    "        \n",
    "df = pd.DataFrame(data)\n",
    "del data, embs_search_np, embs_np, item_emb_map\n",
    "gc.collect()\n",
    "df['event_timestamp'] = pd.to_datetime(df['event_timestamp']).dt.tz_localize(None)\n",
    "df[df['event_type'] != 'search'].to_parquet(\"./data/22_test_view_melted.parquet\",engine='fastparquet', compression=None)\n",
    "df[df['event_type'] == 'search'].to_parquet(\"./data/22_test_search_melted.parquet\",engine='fastparquet', compression=None)\n",
    "df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}