P9_v0.3_wblob.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ac9f54ea-4690-4d65-b23d-5d082d5a12e2",
   "metadata": {},
   "source": [
    "### data creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f39a2e3b-22b3-4711-960d-83cc38d3c508",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cut_data_by_qtime(qtime=0.5):\n",
    "    '''le problème principal est que la liste d articles étaient presque complètes avant la première interaction\n",
    "        en cutant par un quantile du click_timestamp, on diminuer le nombre de clics mais pas le nombre d'articles\n",
    "        et si on cut par un quantile items.created_at_ts, on aura plus d interactions\n",
    "        \n",
    "        dans l ideal il faut cutter les items par un quantile items.created_at_ts\n",
    "                                et les interactions par un quantile click_timestamp\n",
    "                                en virant toutes les interactions avec des items cuttés\n",
    "                                mais on peut renconter un grand saut dans le nombre d interactions'''\n",
    "\n",
    "    cut_click=clics.click_timestamp.quantile(qtime)\n",
    "    #cut_items=items.created_at_ts.quantile(q)\n",
    "    #fusion items et items2vec \"embedding\"\n",
    "    items_=pd.concat([items,pd.DataFrame(items2vec)], axis=1)\n",
    "    #data cut\n",
    "    clics__=clics.loc[clics.click_timestamp<cut_click].copy()\n",
    "    items__=items_.loc[items_.created_at_ts<cut_click].copy()\n",
    "    \n",
    "    #merge clics & items, pour retirer les articles non connus\n",
    "    data=clics__.loc[clics__.click_article_id.isin(set(items__.article_id))].copy()\n",
    "    data.rename(columns={'click_article_id':'article_id'}, inplace = True)\n",
    "    #display(data)\n",
    "    #1 ligne par item interagi pour chaque user\n",
    "    data_map=data[['user_id','article_id']].groupby(['user_id', 'article_id']).size().to_frame().reset_index()\n",
    "    data_map.columns=['user_id','article_id','nb']\n",
    "    #display(data_map)\n",
    "    #extraction item2vec\n",
    "    items2vec_=items__.iloc[:,5:].to_numpy()\n",
    "    #creation des dicos inner_id/raw_id et raw_id/inner_id\n",
    "    dico_inner_raw_iid = dict( (inner_iid,iid) for inner_iid,iid in enumerate(items__.article_id ))\n",
    "    dico_raw_inner_iid = dict( (iid,inner_iid) for inner_iid,iid in enumerate(items__.article_id ))\n",
    "    return data_map, items__, items2vec_, dico_inner_raw_iid, dico_raw_inner_iid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "84a62fd9-e6dc-4472-be4b-1aa3cd82a10b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#globo user data\n",
    "import pandas as pd\n",
    "#article content embedding : np matrix 364047 articles / 250D\n",
    "#embeddings can be used by Neural Networks to represent their content\n",
    "items2vec = pd.read_pickle(r'data/news-portal-user-interactions-by-globocom/articles_embeddings.pickle')\n",
    "#metadata info about 364047 articles\n",
    "items = pd.read_csv('data/news-portal-user-interactions-by-globocom/articles_metadata.csv', delimiter = ',')\n",
    "#all_click\n",
    "for i in range(365):\n",
    "    temp=pd.read_csv('data/news-portal-user-interactions-by-globocom/clicks/clicks/clicks_hour_'+\"%03d\" % i+'.csv', delimiter = ',')\n",
    "    if i==0:\n",
    "        clics=temp\n",
    "    else:\n",
    "        clics=pd.concat([clics, temp])\n",
    "clics=clics.reset_index(drop=True)\n",
    "\n",
    "data_map,items_df, i2vec, dic_ir, dic_ri=cut_data_by_qtime(qtime=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3d3e181a-330f-4e0e-9db2-148adefc34cc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>article_id</th>\n",
       "      <th>nb</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>68866</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>157541</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>96663</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>235840</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>30970</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27919</th>\n",
       "      <td>10721</td>\n",
       "      <td>156543</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27920</th>\n",
       "      <td>10721</td>\n",
       "      <td>207122</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27921</th>\n",
       "      <td>10722</td>\n",
       "      <td>159359</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27922</th>\n",
       "      <td>10723</td>\n",
       "      <td>207122</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27923</th>\n",
       "      <td>10733</td>\n",
       "      <td>207122</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>27924 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       user_id  article_id  nb\n",
       "0            0       68866   1\n",
       "1            0      157541   1\n",
       "2            1       96663   1\n",
       "3            1      235840   1\n",
       "4            2       30970   1\n",
       "...        ...         ...  ..\n",
       "27919    10721      156543   1\n",
       "27920    10721      207122   1\n",
       "27921    10722      159359   1\n",
       "27922    10723      207122   1\n",
       "27923    10733      207122   1\n",
       "\n",
       "[27924 rows x 3 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7213e285-2123-4cb3-b514-36c3f165dc1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#calcul des user profils\n",
    "import numpy as np\n",
    "from sklearn import preprocessing\n",
    "def _build_users_profile(uid, click_df,emb_matrix,dic_ri):\n",
    "    '''calcul du profil embedding pour un uid\n",
    "    à partir des interactions du train'''\n",
    "    #on recupere le dataframe d'interaction pour l'uid ciblé\n",
    "    click_uid_df = click_df.loc[click_df.user_id==uid]\n",
    "    #on recupere les embeddings des items vus par l'uid\n",
    "    user_item_profiles = np.array([emb_matrix[dic_ri[iid]] for iid in click_uid_df['article_id']])\n",
    "    #on recupere le nombre de click sur les articles par l'uid afin d apporter une ponderation à la moyenne des embedding pour le calcul du profil\n",
    "    user_item_strengths = np.array(click_uid_df['nb']).reshape(-1,1) #-1 veut dire unknow dim\n",
    "    #on pondere la localisation embedding de chaque item par le nombre d'interactions puis on somme afin d'obtenir le profile qui est un barycentre\n",
    "    user_item_strengths_weighted_avg = np.sum(np.multiply(user_item_profiles,user_item_strengths), axis=0) / np.sum(user_item_strengths)\n",
    "    user_profile_norm = preprocessing.normalize(np.reshape(user_item_strengths_weighted_avg,(1,250)))\n",
    "    return user_profile_norm\n",
    "\n",
    "def _build_users_profiles(click_df, emb_matrix,dic_ri): \n",
    "    '''calcul des profils de tous les uid sous forme de dic {uid:profil}\n",
    "    à partir des interactions du train'''\n",
    "    user_profiles = {}\n",
    "    for uid in click_df.user_id.unique():\n",
    "        user_profiles[uid] = _build_users_profile(uid, click_df, emb_matrix, dic_ri)\n",
    "    return user_profiles\n",
    "\n",
    "profils=_build_users_profiles(data_map, i2vec,dic_ri)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b99457fd-1841-45b5-974b-10a940ce18d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<surprise.prediction_algorithms.matrix_factorization.SVDpp at 0x1fe4cfc87c0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from surprise import Reader, Dataset,SVDpp, dump\n",
    "\n",
    "#fit du collab filtering\n",
    "'fit du modele SVDpp'\n",
    "reader = Reader(rating_scale=(0,5))\n",
    "#lecture du train par surprise\n",
    "data = Dataset.load_from_df(data_map[['user_id','article_id','nb']], reader)\n",
    "#construction du trainset surprise \n",
    "trainset = data.build_full_trainset()\n",
    "algo = SVDpp(n_epochs=20, lr_all=0.007, reg_all=0.1)\n",
    "algo.fit(trainset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6260d25d-497d-4bea-b29f-da495d35aad1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dump algorithm and reload it.\n",
    "import os\n",
    "local_path = \"./2AzBlob\"\n",
    "#os.mkdir(local_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e23d4245-6e71-4877-af89-c82f6473ab32",
   "metadata": {},
   "outputs": [],
   "source": [
    "#tout en un seul pickle pour essayer de regler les problemes server 500 en prod de l azure function\n",
    "import pickle\n",
    "\n",
    "local_file='all.pkl'\n",
    "file_path = os.path.join(local_path, local_file)\n",
    "file = open(file_path,'wb')\n",
    "\n",
    "pickle.dump(algo, file)\n",
    "pickle.dump(data_map, file)\n",
    "pickle.dump(items_df, file)\n",
    "pickle.dump(i2vec, file)\n",
    "pickle.dump(dic_ir, file)\n",
    "pickle.dump(dic_ri, file)\n",
    "pickle.dump(profils, file)\n",
    "\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10964d10-6bcf-4610-b291-b4a1c41cde50",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "local_file='algo.pkl'\n",
    "file_path = os.path.join(local_path, local_file)\n",
    "pickle.dump(file_path, algo=algo)\n",
    "#on binarise/serialise le numpy array du word embedding\n",
    "local_file='data_map.pkl'\n",
    "file_path = os.path.join(local_path, local_file)\n",
    "with open(file_path, 'wb') as f:\n",
    "    pickle.dump(data_map, f)\n",
    "    \n",
    "local_file='items_df.pkl'\n",
    "file_path = os.path.join(local_path, local_file)\n",
    "with open(file_path, 'wb') as f:\n",
    "    pickle.dump(items_df, f)\n",
    "    \n",
    "local_file='i2vec.pkl'\n",
    "file_path = os.path.join(local_path, local_file)\n",
    "with open(file_path, 'wb') as f:\n",
    "    pickle.dump(i2vec, f)\n",
    "\n",
    "local_file='dic_ir.pkl'\n",
    "file_path = os.path.join(local_path, local_file)\n",
    "with open(file_path, 'wb') as f:\n",
    "    pickle.dump(dic_ir, f)\n",
    "\n",
    "local_file='dic_ri.pkl'\n",
    "file_path = os.path.join(local_path, local_file)\n",
    "with open(file_path, 'wb') as f:\n",
    "    pickle.dump(dic_ri, f)\n",
    "\n",
    "local_file='profils.pkl'\n",
    "file_path = os.path.join(local_path, local_file)\n",
    "with open(file_path, 'wb') as f:\n",
    "    pickle.dump(profils, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "0c348a7b-6d4e-4b72-b552-0c0604c15bef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Azure Blob Storage v12.12.0 - Python quickstart sample\n",
      "\n",
      "Uploading to Azure Storage as blob:\n",
      "\tall.pkl\n"
     ]
    }
   ],
   "source": [
    "import os, uuid\n",
    "from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient, __version__\n",
    "\n",
    "try:\n",
    "    print(\"Azure Blob Storage v\" + __version__ + \" - Python quickstart sample\")\n",
    "        # recuperation de la chaine de connexion au azure blob storage à partir de la variabale environmentale\n",
    "    connect_str = os.getenv('AZURE_STORAGE_CONNECTION_STRING')\n",
    "\n",
    "        # Creation du service_client blob qui permettra de creer un conteneur\n",
    "    blob_service_client = BlobServiceClient.from_connection_string(connect_str)\n",
    "        # Creation d'un nom unique aleatoire grace a la fonction uuid4 pour le conteneur blob\n",
    "    container_name = \"522f43be-48af-4f3d-a2d7-4deba1749d31\" #str(uuid.uuid4())\n",
    "        # Creation du conteneur si il n'existe pas\n",
    "    #container_client = blob_service_client.create_container(container_name)\n",
    "    container_client = blob_service_client.get_container_client(container_name)\n",
    "\n",
    "\n",
    "    toblob_list=['all.pkl']\n",
    "    #toblob_list=['data_map.pkl','items_df.pkl','i2vec.pkl','dic_ir.pkl','dic_ri.pkl','algo.pkl','profils.pkl']\n",
    "\n",
    "    # Create a blob client using the local file name as the name for the blob\n",
    "    for i in toblob_list:\n",
    "        local_file = i\n",
    "        upload_file_path = os.path.join(local_path, local_file)\n",
    "        blob_client = blob_service_client.get_blob_client(container=container_name, blob=local_file)\n",
    "\n",
    "        print(\"\\nUploading to Azure Storage as blob:\\n\\t\" + local_file)\n",
    "\n",
    "        # Upload the created file\n",
    "        with open(upload_file_path, \"rb\") as data:\n",
    "            blob_client.upload_blob(data)\n",
    "\n",
    "\n",
    "except Exception as ex:\n",
    "    print('Exception:')\n",
    "    print(ex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f2fe7f10-015b-4974-9621-62f6a09783ec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Listing blobs...\n",
      "\talgo.pkl\n",
      "\tall.pkl\n",
      "\tdata_map.pkl\n",
      "\tdic_ir.pkl\n",
      "\tdic_ri.pkl\n",
      "\ti2vec.pkl\n",
      "\titems_df.pkl\n",
      "\tprofils.pkl\n"
     ]
    }
   ],
   "source": [
    "print(\"\\nListing blobs...\")\n",
    "#container_client = blob_service_client.get_container_client(container_name)\n",
    "# List the blobs in the container\n",
    "blob_list = container_client.list_blobs()\n",
    "for blob in blob_list:\n",
    "    print(\"\\t\" + blob.name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "0493740f-8348-41a7-8848-6b4c774cd64c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from io import BytesIO\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "9cc25cea-6e94-4c0e-b62c-98e029aab0a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "blob_client = blob_service_client.get_blob_client(container=container_name, blob='all.pkl')\n",
    "hh=(blob_client.download_blob(0).readall())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "9c37c324-e8f0-4fce-9868-3d0d29459fb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from io import BytesIO\n",
    "blob_to_read = BytesIO(hh)\n",
    "#with open(hh, 'rb') as file:\n",
    "with blob_to_read as file:\n",
    "#blob_client = blob_service_client.get_blob_client(container=container_name, blob='all.pkl')\n",
    "#file = open(blob_client.download_blob(0).readall(), 'rb')\n",
    "    algo_ = pickle.load(file)\n",
    "    data_map_ = pickle.load(file)\n",
    "    items_df_ = pickle.load(file)\n",
    "    i2vec_ = pickle.load(file)\n",
    "    dic_ir_= pickle.load(file)\n",
    "    dic_ri_ = pickle.load(file)\n",
    "    profils_= pickle.load(file)\n",
    "#file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "7c20c517-0b56-4c3e-ab1e-3ce9b2bc17b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>article_id</th>\n",
       "      <th>nb</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>68866</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>157541</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>96663</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>235840</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>30970</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27919</th>\n",
       "      <td>10721</td>\n",
       "      <td>156543</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27920</th>\n",
       "      <td>10721</td>\n",
       "      <td>207122</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27921</th>\n",
       "      <td>10722</td>\n",
       "      <td>159359</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27922</th>\n",
       "      <td>10723</td>\n",
       "      <td>207122</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27923</th>\n",
       "      <td>10733</td>\n",
       "      <td>207122</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>27924 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       user_id  article_id  nb\n",
       "0            0       68866   1\n",
       "1            0      157541   1\n",
       "2            1       96663   1\n",
       "3            1      235840   1\n",
       "4            2       30970   1\n",
       "...        ...         ...  ..\n",
       "27919    10721      156543   1\n",
       "27920    10721      207122   1\n",
       "27921    10722      159359   1\n",
       "27922    10723      207122   1\n",
       "27923    10733      207122   1\n",
       "\n",
       "[27924 rows x 3 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_map_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d81df49e-4c95-4fd1-be08-0fa1f5b99fbd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 219 ms\n",
      "Wall time: 222 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "local_file='all.pkl'\n",
    "file_path = os.path.join(local_path, local_file)\n",
    "file = open(file_path,'rb')\n",
    "\n",
    "#blob_client = blob_service_client.get_blob_client(container=container_name, blob='all.pkl')\n",
    "#file = open(blob_client.download_blob(0).readall(), 'rb')\n",
    "algo_ = pickle.load(file)\n",
    "data_map_ = pickle.load(file)\n",
    "items_df_ = pickle.load(file)\n",
    "i2vec_ = pickle.load(file)\n",
    "dic_ir_= pickle.load(file)\n",
    "dic_ri_ = pickle.load(file)\n",
    "profils_= pickle.load(file)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "bb63ddec-4471-44f4-8e28-05f843ba5ab6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 5.61 s\n",
      "Wall time: 22.1 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "#chargement du blob\n",
    "#blob_client = BlobClient.from_connection_string(connection_string, container_name, blob_name)\n",
    "blob_client = blob_service_client.get_blob_client(container=container_name, blob='data_map.pkl')\n",
    "data_map_fb=pickle.loads(blob_client.download_blob(0).readall())\n",
    "blob_client = blob_service_client.get_blob_client(container=container_name, blob='items_df.pkl')\n",
    "items_df_fb=pickle.loads(blob_client.download_blob(0).readall())\n",
    "blob_client = blob_service_client.get_blob_client(container=container_name, blob='i2vec.pkl')\n",
    "i2vec_fb=pickle.loads(blob_client.download_blob(0).readall())\n",
    "blob_client = blob_service_client.get_blob_client(container=container_name, blob='dic_ir.pkl')\n",
    "dic_ir_fb=pickle.loads(blob_client.download_blob(0).readall())\n",
    "blob_client = blob_service_client.get_blob_client(container=container_name, blob='dic_ri.pkl')\n",
    "dic_ri_fb=pickle.loads(blob_client.download_blob(0).readall())\n",
    "blob_client = blob_service_client.get_blob_client(container=container_name, blob='profils.pkl')\n",
    "profils_fb=pickle.loads(blob_client.download_blob(0).readall())\n",
    "blob_client = blob_service_client.get_blob_client(container=container_name, blob='algo.pkl')\n",
    "algo_fb=pickle.loads(blob_client.download_blob(0).readall())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "64dca7a1-4bb3-4eda-923f-05126f320ac3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'522f43be-48af-4f3d-a2d7-4deba1749d31'"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "container_name"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80278f92-48ef-48af-92b3-ceecb861891b",
   "metadata": {},
   "source": [
    "### reco"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "c813a120-f537-4d88-834e-37edb2d01e47",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn import preprocessing\n",
    "import scipy\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from surprise import Reader, Dataset,SVDpp\n",
    "\n",
    "class ContentBasedRecommender:\n",
    "    '''le fit calcule les users profiles'''\n",
    "    \n",
    "    MODEL_NAME = 'Content-Based'\n",
    "    \n",
    "\n",
    "    def __init__(self,data_map,i2vec,dic_ri,dic_ir,profils):\n",
    "        '''constructeur'''\n",
    "        #dico inner raw iid\n",
    "        self.dic_ir = dic_ir\n",
    "        #dico raw inner iid\n",
    "        self.dic_ri = dic_ri\n",
    "        self.items_embedding = i2vec\n",
    "        #les train interactions servent à:\n",
    "            #calculer les profils\n",
    "            #ignorer les interactions dans la reco\n",
    "        self.train_user_interact = data_map\n",
    "        self.user_profiles = profils\n",
    "    \n",
    "    \n",
    "    def get_model_name(self):\n",
    "        '''renvoie le nom du modèle'''\n",
    "        return self.MODEL_NAME\n",
    "    \n",
    "\n",
    "    def _get_similar_items_to_user_profile(self, uid):#, topn=100000):\n",
    "        '''renvoie une liste d items similaires au profil de l'uid\n",
    "            liste de couples (raw iid, cosine) triés par cosine decroissant'''\n",
    "        #on calcul distances profil uid vs articles : sous forme 2D 1 ligne / nombre iid colonnes dans l'ordre inner du wemb\n",
    "        cosine_similarities = cosine_similarity(self.user_profiles[uid], self.items_embedding)\n",
    "        #on tri par distance: argsort donne les indices par scores croissants /flatten pour passer en 1D\n",
    "        #on recupère un array des inner_iid de taille topn : les plus proches du profil sont à droite\n",
    "        similar_indices = cosine_similarities.argsort().flatten()#[-topn:]\n",
    "                                                        ################################################################################\n",
    "        #on tri : sorted(iterable, key=None, reverse=False)\n",
    "        # l'iterable est une liste de couples (raw_iid, cosine similarites)\n",
    "        #on tri par similarité décroissante\n",
    "        similar_items = [(self.dic_ir[i], cosine_similarities[0,i]) for i in similar_indices[::-1]]\n",
    "        return similar_items\n",
    "    \n",
    "    #methode qui recommande en ignorant les articles deja vus\n",
    "    def recommend_items(self, uid, topn=5):\n",
    "        '''renvoie une liste de reco similaires au profil de l uid\n",
    "            triés par cosine decroissant\n",
    "             sans les interactions du train\n",
    "                sous forme de dataframe'''\n",
    "        #on recupere la liste brute de 1200 reco classés par cosine decroissant\n",
    "        similar_items = self._get_similar_items_to_user_profile(uid)\n",
    "        #on recupere la liste des iid a ignorer car deja vus\n",
    "        #items_to_ignore = self.train_user_interact.loc[self.train_user_interact.user_id==uid,\"article_id\"].tolist()\n",
    "        items_to_ignore = set(self.train_user_interact.loc[self.train_user_interact.user_id==uid].article_id)\n",
    "        #on enleve les deja vus\n",
    "        #similar_items_filtered = list(filter(lambda x: x[0] not in items_to_ignore, similar_items))\n",
    "        recommendations_df = pd.DataFrame(similar_items, columns=['article_id', 'cb_cosine_with_profile'])\n",
    "        reco = recommendations_df.loc[recommendations_df.article_id.isin(items_to_ignore)==False].copy()\n",
    "        if topn>0:\n",
    "            reco=reco.head(topn)\n",
    "        return reco\n",
    "\n",
    "\n",
    "class CollabFiltRecommender:\n",
    "    \n",
    "    MODEL_NAME = 'Collaborative-Filtering-SVDpp'\n",
    "    \n",
    "    #constructeur\n",
    "    def __init__(self,algo,data_map):\n",
    "        '''constructeur'''\n",
    "        #les train interactions servent à:\n",
    "            #calculer les profils\n",
    "            #ignorer les interactions dans la reco\n",
    "        #self.train_user_interact = data_map\n",
    "        self.algo=algo\n",
    "        self.train_user_interact = data_map\n",
    "        \n",
    "    #methode qui renvoie le nom du modele    \n",
    "    def get_model_name(self):\n",
    "        return self.MODEL_NAME\n",
    "        \n",
    "    def recommend_items(self,uid,topn=5):\n",
    "        '''renvoie une liste de prediction pour un uid\n",
    "            le SVD ne peut sortir des prédictions que pour les uid et iid dispo dans le train'''\n",
    "        #prediction fonctionne avec les raws id\n",
    "        #https://towardsdatascience.com/difference-between-apply-and-transform-in-pandas-242e5cf32705\n",
    "        iid_to_ignore=set(self.train_user_interact.loc[self.train_user_interact.user_id==uid].article_id)\n",
    "        items2pred=pd.DataFrame(set(self.train_user_interact.article_id)-iid_to_ignore,columns=['article_id'])\n",
    "        items2pred['pred']=items2pred['article_id'].apply(lambda x:self.algo.predict(uid=uid, iid=x)[3])\n",
    "        #pred['detail']=pred['article_id'].apply(lambda x:cf.predict(uid=uid, iid=x)[4])\n",
    "        if topn==0:\n",
    "            recommendations_df=items2pred.loc[:,['article_id','pred']].sort_values(by='pred', ascending=False)\n",
    "        else:\n",
    "            recommendations_df=items2pred.loc[:,['article_id','pred']].sort_values(by='pred', ascending=False).head(topn)\n",
    "\n",
    "        return recommendations_df\n",
    "\n",
    "class PopularityFiltRecommender:\n",
    "    \n",
    "    MODEL_NAME = 'Popularity-Filtering'\n",
    "    \n",
    "    #constructeur\n",
    "    def __init__(self,data_map):\n",
    "        '''constructeur'''\n",
    "        self.train_user_interact = data_map\n",
    "        \n",
    "    #methode qui renvoie le nom du modele    \n",
    "    def get_model_name(self):\n",
    "        return self.MODEL_NAME\n",
    "    \n",
    "    #fit du modèle\n",
    "    def fit(self):\n",
    "        self.raw_reco=self.train_user_interact[['nb','article_id']].groupby(['article_id']).sum().sort_values(by=['nb'],ascending=False).reset_index()\n",
    "\n",
    "    def recommend_items(self,uid,topn=5):\n",
    "        '''renvoie une liste de prediction pour un uid'''\n",
    "        if topn==0:\n",
    "            recommendations_df=self.raw_reco\n",
    "        else:\n",
    "            recommendations_df=self.raw_reco.head(topn)\n",
    "        return recommendations_df\n",
    "\n",
    "class HybridRecommender:\n",
    "    \n",
    "    MODEL_NAME = 'Hybrid-Filtering'\n",
    "    \n",
    "    #constructeur\n",
    "    def __init__(self,data_map,i2vec,dic_ri,dic_ir, profils, algo):\n",
    "        '''constructeur'''\n",
    "        self.train_user_interact = data_map\n",
    "        self.dic_ir = dic_ir\n",
    "        self.dic_ri = dic_ri\n",
    "        self.items_embedding = i2vec\n",
    "        self.user_profiles = profils\n",
    "        self.algo=algo\n",
    "    \n",
    "    def get_model_name(self):\n",
    "        return self.MODEL_NAME\n",
    "    \n",
    "    #fit du modèle\n",
    "    def fit(self):\n",
    "        '''le fit consiste a fitter les sous modèles'''\n",
    "        self.cf_model = CollabFiltRecommender(self.algo,self.train_user_interact)\n",
    "        self.cb_model = ContentBasedRecommender(self.train_user_interact,self.items_embedding,self.dic_ri,self.dic_ir,self.user_profiles)\n",
    "        self.pf_model = PopularityFiltRecommender(self.train_user_interact)\n",
    "        self.pf_model.fit()\n",
    "        \n",
    "    def recommend_items(self,uid,topn=5):\n",
    "        '''renvoie une liste de prediction pour un uid'''\n",
    "        #si uid pas connu du train: seul le popularity recommander fonctionne dans notre cas\n",
    "        #on pourrait également concevoir un social reco basé sur un social profil, encore faudrait il avoir assez d 'infos'\n",
    "        if uid not in set(self.train_user_interact.user_id):\n",
    "            #parfait quand l'uid n'est pas connu du train, n a pas  d'interactions\n",
    "            reco=self.pf_model.recommend_items(uid,0)\n",
    "        else:\n",
    "                # content based : permet de donner des iid qui n'ont pas d'interactions mais qui étaient disponibles\n",
    "            reco_cb=self.cb_model.recommend_items(uid,0)\n",
    "                # pour la normalisation, le but est de garder les meilleurs, donc les outliers positifs\n",
    "                #il vaut mieux dans ce cas diviser par le max que par une standard deviation \n",
    "                #qui ne rendra pas forcement les outliers similaires entre facteurs pour une pondéaration\n",
    "            #reco_cb['norm_cb']=(reco_cb.cb_cosine_with_profile-reco_cb.cb_cosine_with_profile.median())/(reco_cb.cb_cosine_with_profile.max()-reco_cb.cb_cosine_with_profile.median())\n",
    "            reco_cb['rank_cb']=reco_cb.cb_cosine_with_profile.rank(ascending=False)\n",
    "                # donne de bons resultats mais necessite des uid et des iid qui ont des interactions dans le train\n",
    "            reco_cf=self.cf_model.recommend_items(uid,0)\n",
    "            #reco_cf['norm_cf']=(reco_cf.pred-reco_cf.pred.median())/(reco_cf.pred.max()-reco_cf.pred.median())\n",
    "            reco_cf['rank_cf']=reco_cf.pred.rank(ascending=False)\n",
    "                #comment les pondérer? 80/20 normalisé par exemple?\n",
    "                #en gros la pondération est anecdotique: on privilegie CF et quand on a pas de CF, on met la note de CB\n",
    "            reco = reco_cb.merge(reco_cf,how='outer', on='article_id')\n",
    "                #on fill les norm_cf vides (iid inconnu du train) par norm_cv avant pondération\n",
    "                #on pourait également filler les iid peu interagis afin de favoriser le content dans ce cas la\n",
    "            reco.loc[reco['rank_cf'].isnull(),'rank_cf'] = reco['rank_cb']\n",
    "            reco['rank']=0.99999*reco['rank_cf']+0.00001*reco['rank_cb']\n",
    "            \n",
    "            reco=reco.sort_values(by='rank', ascending=True)\n",
    "\n",
    "\n",
    "        if topn==0:\n",
    "            recommendations_df=reco\n",
    "        else:\n",
    "            recommendations_df=reco.head(topn)\n",
    "            \n",
    "        return recommendations_df.reset_index(drop=True)\n",
    "        #return recommendations_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "0327d6c1-6229-4538-951f-12607df95296",
   "metadata": {},
   "outputs": [],
   "source": [
    "hr_model = HybridRecommender(data_map_fb,i2vec_fb,dic_ri_fb,dic_ir_fb,profils_fb,algo_fb)\n",
    "hr_model.fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "2226c1e2-f513-467b-9757-df8b0eafbb3b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 875 ms\n",
      "Wall time: 785 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[285781, 97948, 207742, 353724, 232088]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "hr_model.recommend_items(7).article_id.to_list()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "7bf038d1-2daa-4ffe-bc8f-f9491cf4a10b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4    1\n",
       "Name: nb, dtype: int64"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_map_fb.loc[(data_map_fb.user_id==2) & (data_map_fb.article_id==30970)].nb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3cdc7436-cd16-4c79-9855-2311b8c92534",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.0\n"
     ]
    }
   ],
   "source": [
    "import pickle \n",
    "print(pickle.format_version)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbc84f94-1d72-485a-84f4-6e3a727920d0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env",
   "language": "python",
   "name": "env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}