ScaledSVD_YaMus.ipynb

{
 "cells": [
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "%env MKL_NUM_THREADS=20\n",
    "%env OMP_NUM_THREADS=20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from ipypb import track\n",
    "\n",
    "from polara.evaluation import evaluation_engine as ee\n",
    "from polara.evaluation.pipelines import (find_optimal_svd_rank,\n",
    "                                         find_optimal_config,\n",
    "                                         set_config)\n",
    "\n",
    "from scaledsvd import ScaledSVD, ScaledSVDItemColdStart\n",
    "from data_preprocessing import (get_yahoo_music_data,\n",
    "                                get_similarity_data,\n",
    "                                prepare_data_model,\n",
    "                                prepare_cold_start_data_model)\n",
    "from utils import (report_results, save_results,\n",
    "                   apply_config, print_data_stats,\n",
    "                   save_training_time, save_cv_training_time)\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from polara.recommender import defaults\n",
    "defaults.memory_hard_limit = 15 # allowed memory usage during recommendations generationa\n",
    "max_test_workers = 6 # use this manyparallel thread for evaluation each using up to {memory_hard_limit} Gb of RAM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 42"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment_name = 'scaledsvd'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiment setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_labels = ['YaMus']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "ranks_grid = [1, 15, 30, 50, 75, 100, 150, 200, 250, 300, 400,\n",
    "              500, 750, 1000, 1250, 1500, 2000, 2500, 3000]\n",
    "svd_ranks = {'YaMus': ranks_grid}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaling_params = [0.2, 0.4, 0.6]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "topk_values = [1, 3, 10, 20, 30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_metric = 'mrr'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dict = dict.fromkeys(data_labels)\n",
    "meta_dict = dict.fromkeys(data_labels)\n",
    "similarities = dict.fromkeys(data_labels)\n",
    "sim_indices = dict.fromkeys(data_labels)\n",
    "feature_idx = dict.fromkeys(data_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data = [data_dict, similarities, sim_indices, meta_dict]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Yahoo Music"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "lbl = 'YaMus'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_dict[lbl], meta_dict[lbl] = get_yahoo_music_data('/data/recsys/yahoo_music/yamus_train0_rating5.gz',\n",
    "                                                      meta_path='/data/recsys/yahoo_music/yamus_attrs.gz',\n",
    "                                                      implicit=True,\n",
    "                                                      pcore=5,\n",
    "                                                      filter_data={'genreid': [0]}, # filter unknown genre\n",
    "                                                      filter_no_meta=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "similarities[lbl], sim_indices[lbl], feature_idx[lbl] = get_similarity_data(meta_dict[lbl])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(meta_dict[lbl].applymap(len).sum(axis=1)==0).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "YaMus\n",
      "{'userid': 183003, 'songid': 134059}\n",
      "density 0.09740952587383789\n",
      "similarity matrix density 0.4576464914574314\n"
     ]
    }
   ],
   "source": [
    "print_data_stats(data_labels, all_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Standard experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_recommender_models(data_label, data_models, config):\n",
    "    data_model = data_models[data_label]\n",
    "    svd = ScaledSVD(data_model)\n",
    "    svd.max_test_workers = max_test_workers\n",
    "    models = [svd]\n",
    "    apply_config(models, config, data_label)\n",
    "    return models\n",
    "\n",
    "def fine_tune_scaledsvd(model, ranks, scale_params, label, record_time=False):\n",
    "    param_grid = [(s, r) for s in scale_params for r in reversed(list(sorted(ranks)))]\n",
    "    param_names = ('col_scaling', 'rank')\n",
    "    model.max_test_workers = max_test_workers\n",
    "    best_svd_config, svd_scores = find_optimal_config(model, param_grid, param_names,\n",
    "                                                      target_metric,\n",
    "                                                      return_scores=True,\n",
    "                                                      force_build=False,\n",
    "                                                      iterator=lambda x: track(x, label=label))\n",
    "    model_config = {model.method: dict(zip(param_names, best_svd_config))}\n",
    "    model_scores = {model.method: svd_scores}\n",
    "    try:\n",
    "        if record_time:\n",
    "            max_rank = max(ranks)\n",
    "            save_training_time(f'{experiment_name}_rank_{max_rank}', model, svd_scores.xs(max_rank, level='rank').index, label)\n",
    "    finally:\n",
    "        return model_config, model_scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = {}\n",
    "scores = {}\n",
    "data_models = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:0; max-width:15ex; vertical-align:middle; text-align:right\"></span>\n",
       "<progress style=\"width:60ex\" max=\"1\" value=\"1\" class=\"Progress-main\"/></progress>\n",
       "<span class=\"Progress-label\"><strong>100%</strong></span>\n",
       "<span class=\"Iteration-label\">1/1</span>\n",
       "<span class=\"Time-label\">[08:08:36<08:08:36, 29315.91s/it]</span></div>"
      ],
      "text/plain": [
       "\u001b[A\u001b[A\u001b[2K\r",
       " [████████████████████████████████████████████████████████████] 1/1 [08:08:36<08:08:36, 29315.91s/it]\u001b[B"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:15ex; max-width:15ex; vertical-align:middle; text-align:right\">YaMus</span>\n",
       "<progress style=\"width:45ex\" max=\"57\" value=\"57\" class=\"Progress-main\"/></progress>\n",
       "<span class=\"Progress-label\"><strong>100%</strong></span>\n",
       "<span class=\"Iteration-label\">57/57</span>\n",
       "<span class=\"Time-label\">[08:08:05<00:51, 513.78s/it]</span></div>"
      ],
      "text/plain": [
       "\u001b[A\u001b[2K\r",
       "          YaMus [█████████████████████████████████████████████] 57/57 [08:08:05<00:51, 513.78s/it]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for label in track(data_labels):\n",
    "    data_models[label] = prepare_data_model(label, *all_data, seed)\n",
    "    config[label], scores[label] = fine_tune_scaledsvd(ScaledSVD(data_models[label]),\n",
    "                                                       svd_ranks[label],\n",
    "                                                       scaling_params,\n",
    "                                                       label, record_time=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/py36/lib/python3.6/site-packages/pandas/plotting/_core.py:1001: UserWarning: Attempted to set non-positive left xlim on a log-scaled axis.\n",
      "Invalid limit will be ignored.\n",
      "  ax.set_xlim(left, right)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEXCAYAAABBFpRtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl81NW5+PHPM5PJvm+QlbDvEJRFVMBdRCq1VgVttbjVW7mtXW5r21tr7a7e+tNqr2KtuNS6tb1aN9qKCwoiIPumYQ9bICwBAiQzc35/zHfCECbJJJmZ70zyvF+vvJjMfOc7ZzLDM2eec85zxBiDUkqp7sFhdwOUUkpFjwZ9pZTqRjToK6VUN6JBXymluhEN+kop1Y1o0FdKqW5Eg75SSnUjGvRVlyYifxaRPzW7bpKI1IpIUSv3SxARIyI7RcQZcH2idV93JNutVKRo0Fdd3TeBKSJyMYCIJANPAN81xuwK4f6HgUsCfp8K7At7K5WKEg36qkszxtQC/wnMFpE04KfARmPMHBEZLyIfi8hBEdklIg+LiKvZKZ4Fbgj4/QbgmcADRKRaRM4L+P0XIjLHupwqIs9b3w4OisgnIpIf/meqVGg06KsuzxjzMrAU+AtwG/B16yY38C0gHzgHmBxwm9/fgAtEJFNE8oCzgNfb8fAzgVSgFMgDvgEc79gzUarzEuxugFJRcgewEfixMWYbgDFmccDtm0RkNjAJeCTg+nrgLeBqIAX4O3CiHY/biO9DpZ8xZhWwpMPPQKkw0KCvugVjzB4R2Qes8V8nIoOA/wHOxNcbTwAWBbn7M/jSQin4vhm0xxygGHhJRDLxpYv+2xijA8HKFpreUd3Z48BqfL3wTOBuQIIc9y7QC8g2xiwMcvtRfB8afj39F4wxDcaYe4wxg4FzgSuB68PUfqXaTYO+6s4ygEPAUREZzOn5fACMr/74VOCLLZxnOTDdmuY5FviS/wYRuUBEhomIA6jDl+7xhPE5KNUuGvRVd/Zd4EZ80zIfB15s6UBjzGpjzNoWbv4xMAg4CPwEeD7gtmJ8g8F1+FJL/8Y3oKyULUQ3UVFKqe5De/pKKdWNaNBXSqluRIO+Ukp1Ixr0lVKqG4m5xVn5+fmmoqLC7mYopVRcWbp06T5jTEFbx8Vc0K+oqGDJEl2prpRS7SEiW0M5TtM7SinVjWjQV0qpbiSkoC8ik0Vkg4hUichdQW5PEpEXrdsXiUiFdX2FiBwTkeXWz2Phbb5SSqn2aDOnb20V9yhwMVANLBaR15otSb8ZOGCM6Sci04HfAtdat200xlR2ppGNjY1UV1dz/LiWIQ+H5ORkSktLcbma7xeilOrqQhnIHQtUGWM2AYjIC8A0IDDoTwPusS6/AjwiIsGqFXZIdXU1GRkZVFRUEMbTdkvGGGpra6murqZ37952N0cpFWWhpHdKgO0Bv1db1wU9xqoTfgjfLkEAvUVkmYi8LyITgj2AiNwmIktEZMnevXtPu/348ePk5eVpwA8DESEvL0+/NSnVTYUS9INF2uZV2lo6ZhdQbowZBXwHeN7aSOLUA42ZbYwZbYwZXVAQfJqpBvzw0b+lUt1XKEG/GigL+L0U2NnSMSKSAGQB+40xJ6yNqTHGLMW3Xd2AzjZaKRW/ag7rt0w7hRL0FwP9RaS3iCQC04HXmh3zGr665ABfBuYZY4yIFFgDwYhIH6A/sCk8TY8up9NJZWUlw4YN4+qrr6a+vj4s592zZw9Tp05l5MiRDBkyhClTpgDQu3dvNmzYcMqxd955J/fddx/vvfceWVlZjBo1ioEDBzJx4kRef709e3UrZY91u+oY+8t3eGPlLrub0m21GfStHP0sYC6wDnjJGLNGRO4VkSusw54E8kSkCl8axz+tcyKwUkRW4Bvgvd0Ysz/cTyIaUlJSWL58OatXryYxMZHHHgt99qnb3fJ2qHfffTcXX3wxK1asYO3atfzmN78BYPr06bzwwgtNx3m9Xl555RWuvdY3KWrChAksW7aMDRs28PDDDzNr1izeeeedDj47paJj3xHfnvK/enMdxxp0AzE7hDRP3xjzpjFmgDGmrzHml9Z1dxtjXrMuHzfGXG2M6WeMGeuf6WOM+asxZqgxZqQx5gxjzD8i91SiZ8KECVRVVbFlyxaGDRvWdP0DDzzAPffcA8B5553Hj370IyZNmsRDDz3E3r17ueqqqxgzZgxjxozho48+AmDXrl2UlpY2nWPEiBEAzJgx45Sg/8EHH1BRUUGvXr1Oa09lZSV33303jzzyCAAvv/wyw4YNY+TIkUycODHsz1+pjnJ7fMOBOw4eY/YHcfmlP+7FXO2dtvzsH2tYu7MurOccUpzJT78wNKRj3W43b731FpMnT27z2IMHD/L+++8DcN111/Htb3+bc889l23btnHppZeybt067rjjDq699loeeeQRLrroImbOnElxcTEjRozA4XCwYsUKRo4cyQsvvMCMGTNafKwzzjiD+++/H4B7772XuXPnUlJSwsGDB0N6XkpFQ4PHC0D/wnT+9/0qrh5dSnF2is2t6l60DEOIjh07RmVlJaNHj6a8vJybb765zfv4UzEA//73v5k1axaVlZVcccUV1NXVcfjwYS699FI2bdrErbfeyvr16xk1ahT+aav+3r7b7ebVV1/l6quvbvGxAre9POecc/ja177GE088gcejX6FV7PD39H80ZTDGwG/eWm9zi7qfuOvph9ojDzd/Tj9QQkICXq+36ffmc9/T0tKaLnu9XhYuXEhKyum9mtzcXK677jquu+46pk6dygcffMBVV13FjBkzuOSSS5g0aRIjRoygsLCwxfYtW7aMwYMHA/DYY4+xaNEi3njjDSorK1m+fDl5eXkt3lepaGm0evq98lL5+sQ+PDyvihvG92J0Ra7NLes+tKffCT169KCmpoba2lpOnDjR6gyaSy65pCnnDjR9gMybN69pJtDhw4fZuHEj5eXlAPTt25e8vDzuuuuuVlM7K1eu5Oc//zl33HEHABs3bmTcuHHce++95Ofns3379hbvq1Q0+YO+y+ng9vP60jMzmZ/9Yy1eb/OlPypSNOh3gsvl4u6772bcuHFMnTqVQYMGtXjsww8/zJIlSxgxYgRDhgxpmv2zdOlSRo8ezYgRIxg/fjy33HILY8aMabrfjBkzWL9+PVdeeeUp55s/f37TlM077riDhx9+mAsvvBCA//qv/2L48OEMGzaMiRMnMnLkyAg8e6Xar9FK77icDlITE/jhlEGs2nGIN1frFM5okcBccCwYPXq0ab6Jyrp165pSFyo89G+q7PDMwi3c/eoalv73ReSlJ+H1Gibe/y7luak8f+tZdjcvronIUmPM6LaO056+Uipq/D39BKcv9Dgcwoyx5SzYWMvmfUftbFq3oUFfKRU1/px+ovNk6Ll6dCkJDuEvn2yzq1ndStwE/VhLQ8Uz/Vsqu7itoJ/gPFn0rzAjmYsG9+CVpdWccOsU40iLi6CfnJxMbW2tBqsw8NfTT05Otrspqhtq8Kd3HKdWer1uXDn7jzYwd80eO5rVrcTFPP3S0lKqq6sJVmtftZ9/5yylos3t8eJyymnlvc/tl09ZbgrPL9rKFSOLbWpd9xAXQd/lcukuT0p1AY0eLwmO0xMMDocwfUw598/dwKa9R+hTkG5D67qHuEjvKKW6hkaPweUMvomPDuhGhwZ9pVTUuL1eXM7gYacwI5mLh/gGdI836oBupGjQV0pFTaPbtBj0Aa4ZXcaB+kY+3lQbxVaFx7pddZzzm3kMufttht8zl417j9jdpKA06CuloqbR6z1lumZzZ/XJI9HpYMHG+Av6n+05zI6Dx7h8eBHHGz38ZVFspqk06CulosaX02857KQkOjmjVzYffr4viq0KD49VNO6O8/tx0eAe/H3ZDhrc3jbuFX0a9JVSUeOfstmac/vls3ZXHfuPNkSpVeHhD/pOh3DN6DJqjzYwb32Nza06XcxN2Wz0eNlx8FiH75+U4CAvLfG0ecBKKfu1NGUz0Dn98nngn5+xYOM+po6Inzn7XmvxqMMhTOifT4/MJF5esp3Jw3ra3LJTxVzQX7/7MOf8Zl6nzpGa6KQ8N5WKvDR65Vv/5qbSKz+NosxkHA79QFDKDo0egyuh9aA/vCSLjOQEPqqKr6BvVZjAKUKC08FVZ5Ty2Psbqak7TmFm7KyAj7mgX5qdwq+vGtHh+9c3uNm6v55ttfV8XnOYeetrmvblBEhMcFCWk+L7IMhLo1deKr3yfB8MJTkpreYblVKd4/Z6cbXR6UpwOhjfJ48Pq+Irr+9p6un7fp8yvIg/vLeRhZtqmVZZYmPLThVzQT8nLZFrxpSF7Xwer2F33XG27jvKltp6tu4/ytZ99WypPcrCTbXUN5ycD+x0CCXZKU0fAr4PhDQq8lIpy00l2eUMW7uU6o7amrLpd06/fP65dg/bauspz0uNQss6z7/7l9NKLQ/qmUGKy8mybQc16EeTP5CXZKdwdr9TbzPGsPfICbbW1rNl31G27a/3fTDUHuXV5TuoO+4+5fiirGTfB0FuQNrI+mBIT+ryf0qlOq3R6yXJ1fb/lXP65QPw0cZ9lOeVR7pZYRE4kAu+bywjSrNYtv2gnc06TbeOVCJCYUYyhRnJjAmyMfPB+oamD4Gttb5vB1tr63lnfQ37jpw45dj89ERfuijX+naQn9o0rpCd6tKBZaXwDeSG0tPvW5BGz8xkPqzax4yx8RH0/QO5gf/XR5Xn8OSHmzje6ImZTEG3DvptyU5NpDI1kcqy7NNuO3LCzdbao2yrrT/lg+HjTbX8bdmOU47NTE5oGj9onjYqyEjSDwTVbbhbqb0TSEQ4p18+89bvwes1cTH5onlPH+CM8mwe8xjW7DzEmb1O71jaQYN+B6UnJTC0OIuhxVmn3Xa80UP1gXq2WGMH/rTRqh2HeGv17qY3B0CKy3nKYHJ5wAdDUVbKKW8gpeJdg8fbtFViW87tn8dfP61m7a46hpWc/v8s1vj/WzsDOnGV5b4O47JtBzXod2XJLif9CjPoV5hx2m2NHi87Dx475dvB1tqjbNx7lHc37D1lBV+i00FpbkrTh8DwkiymDC+Kma+JSrWX22NO2SqxNWf3tfL6VfviJOifOnsHfEXkSnNSWLYtdvL6GvSjzOV0WKmeNKDglNu81kyjLc3SRltq61m0qZanPtrCr95cz8xzKrh+XDnZqYn2PAmlOsi3OCu0b689MpPpX5jOh1X7+PqkvhFuWed5ms3e8RtVnsOSLfvtaFJQGvRjiMMhFGenUJydwtnN3uPGGBZsrGX2B5u4f+4GHn23imtGl3Hzub0py42PKW1KhbI4K9CFg3vwxPxN7Dh4jJLslAi2rPOC5fTBl9f/x4qdMfMcdCVSnPAPbD1901jevnMCk4f15LmPtzLp/neZ9fynrKo+ZHcTlWpTKIuzAt0wvhcCPPXh5sg1Kky8xiDCaRMzJg7wfaN/Y+VOO5p1Gg36cWhQz0x+d00l839wPrdO6MP7G/byhUc+ZPrshby7vqZpkYhSsabRHfpALkBxdgpTRxTxl0+2cehYYwRb1nkerzkttQPQtyCdUeXZvLK0GmPs/7+pQT+OFWWl8MMpg1nwwwv48ZTBbK2tZ+acxVz6/z7gpSXbOeHW3YdUbGn0hrYiN9AtE/pwtMHDCzG+jaLHtDy19Oozy/hszxFW7bD/G7kG/S4gI9nFrRP78MH3z+fBa0fidAjff2UlE377Ln94ryrme0iq+2gMobRyc8NKsji7bx5PfbQlJuvT+3lb6OkDXD6iiKQEBy8vqY5yq04XUtAXkckiskFEqkTkriC3J4nIi9bti0Skotnt5SJyRES+F55mq2BcTgdXjirlrW9N4JmbxjKwZwb3vb2Bs3/9Dj9/fW2nSlYr1Vker8EYOlTU8NaJfdhdd5w3VsVGXjwYj/f0QVy/rBQXlw7tyavLd9j+DbzNv76IOIFHgcuAIcAMERnS7LCbgQPGmH7Ag8Bvm93+IPBW55urQiEiTBxQwLM3j+ONb57LJUN78vSCLUy8712+9cIyVsfAV0zV/TRa1W5b2y6xJecNKKB/YTqzP9gcE3nxYLzG0NoY9eRhPak77mbdrsPRa1QQoXzkjgWqjDGbjDENwAvAtGbHTAOeti6/Alwo1hC2iHwR2ASsCU+TVXsMLc7iwWsr+eD75zPz7Ar+vXYPU3//Idf/8WPe/2xvzP4HUl2PP+iHujgrkIhw64Q+rNtVx8IY3TTd4zWtrqAfUepbYLaq2t6FWqH89UuA7QG/V1vXBT3GGOMGDgF5IpIG/AD4WWsPICK3icgSEVmyd+/eUNuu2qE4O4X/njqEBT+8kLsuG0RVzRFu/NMnXPbQfP66tDqmc6Wqa3B7fB2MUBdnNXdFZTGpiU7eWrU7nM0KG49pPeiXZKeQl5bICpunV4cS9IM9i+bdw5aO+RnwoDHmSGsPYIyZbYwZbYwZXVBQ0NqhqpOyUlzcPqkv879/AQ9cPRJj4Lsvr2Dife/y+PsbqTuug74qMk6mdzo2fyTZ5bSKsNXE5DdUr9fgaKV4oogwojSLlXHQ068GAnc1KQWaj6Y0HSMiCUAWsB8YB9wnIluAO4EficisTrZZhUFigoMvn1nK23dOYM7MMfQpSOPXb63n7F/P41dvrmPXIR30VeHVaK0f6Uh6x+/8gYXsOHiMqppW+5G2aCu9AzCiNJuqmiMcPeFu9bhICqUMw2Kgv4j0BnYA04Hrmh3zGnAjsBD4MjDP+D6KJ/gPEJF7gCPGmEfC0G4VJiLCeQMLOW9gIauqDzF7/iae/HAzf/pwM1eMLObWiX0YXJRpdzNVF9Do7vhArt/5g3yZgHnra+jf4/SChnbymNZ7+gAjy7LwGli94xDj+uRFqWWnavMj18rRzwLmAuuAl4wxa0TkXhG5wjrsSXw5/CrgO8Bp0zpV7BtemsXvZ4zive+dx1fH9+LtNbu57KH5fPXJRXz4+b6Y/Eqt4ofb6wv6ndmHuigrhcFFmcxbXxOuZoWNN8SePmDrIq2QCq4ZY94E3mx23d0Bl48DV7dxjns60D5lg7LcVH76haHceeEAnlu0lTkLtvCVJxcxqGcGV1QWc/nwIqtKqFKha3D7Og3tXZzV3PkDC3j8g00cOtZIVoorHE0LC49peZ6+X356EiXZKbYO5uqKXNWirFQXd5zfjw9/cD73XTWCZJeT+97ewKT73+Pyh+fz6LtVbNl31O5mqjgRjp4+wAWDCvF4DR9+vi8czQob30Bu28fZPZirpZVVm5ISnFwzpoxrxpSx4+Ax3lq1i9dX7uL+uRu4f+4GhhZnMmV4EZcPL6IiX78BqOAa/VM2Oxn0R5XnkJ3qYt76Gi4fURSOpoVFKAO5AEOLM3lr9W6OnnCTlhT9EKxBX7VLSXYKt0zowy0T+jR9ALyx6uQHwJCiTC4foR8A6nT+KZvtKa0cjNMhTBpQwPuf1cTU/rmhDOQC9CtMB2DT3qMML43+jmAa9FWHhfoBMGV4Eb31A6Db8y/Oas8mKi05f2Ahry7fycodh6gsy+70+cIhlIFcOBn0P685rEFfxa+2PgAGF2UyVT8AurWmxVlh6JlPGlCAQ+Dd9TWxE/RD7On3yksjwSG2rTXQoK/CLvADYOfBY7wZ5APg8uE9mTK8iD4F6XY3V0VJU3qnkzl9gJy0RCrLsnnvs718++IBnT5fOHgMIaWaXE4HFflpGvRV11Qc5APgzVW7eOCfn/HAPz9r+gCYVlmie/12cf6B3HAEfYBz++XzyLtV1B1vJDPZ/qmbvnr6oR3bryCdz/bYU21Tp2yqqPF/APztG+ew4K4L+O/LB5PicvDAPz/jwt+9z0P//tz2WuMqck5O2QzPwOtZffPwGli8eX9YztdZoc7eAV9ef+v+elve7xr0lS0CPwA+/MH5XDq0Jw/++zMue2g+H8do6VzVOeHu6Z9RnkNigoOFG2Pj/RLq7B2A/j3S8XgNW/bVR7hVp9Ogr2xXmpPK72eMYs7MMTR6vEyf/THfe3kF+4822N00FUad2UQlmGSXkzPKs2Omvn6os3fAt1k6YEteX4O+ihnnDSzkn3dO4hvn9eX/lu3gwv95j1eWVmvNny7CHcaBXL+z++azdlcdB+vt7yC0VU8/UN+CdEQ06CtFSqKT708exBvfnECfgnS+9/IKZjzxMRv3xl4pXdU+Df70jiN8YWd83zyMgY832Z/Xb6uefqCURCelOSl8XhP9wVwN+iomDeyZwctfH8+vrhzO2p11XPb/5vPgvz7jeKMO9Marpp5+QvhW0I4szSbF5YyJcaD29PQB+uSns9mG2lUa9FXMcjiE68aV8853z+Oy4T156J3PmfLQfBZsjK1CWyo0JxdnhS/sJCY4GF2RExODuR4vIff0AUpyUth5MPqbFWnQVzGvICOJh6aP4pmbxuIxhuueWMR3XlpO7ZETdjdNtcPJ2TvhrZUzvm8eG/YcZp/N7wffQG7ox5dkp3CgvpH6hujuoqVBX8WNiQMKmHvnRGad349/rNjJhb97n5eWbNeB3jjh9npJcAjSjt5wKMZbO1DZneJpb3qnJDsFgJ0Hj0eqSUFp0FdxJdnl5HuXDuSNb06gf2E6339lJdfO/pgqGwbEVPs0ekzYpmsGGl6SRXpSgu0pnvYM5IJvrQoQ9RSPBn0Vlwb0yODF28bz26uGs2H3YS57aD7/888NOtAbwxo93rBO1/RLcDoY2zvX9vn67e3pF2cnAxr0lQqZwyFcO6acd747iakjivn9vCq+9IcFbK3V3bxiUaSCPsDY3rls2nuUAzYu6PN4Dc529PR7ZCbjEA36SrVbfnoSD15byZM3jmbHwWNMffhD3l692+5mqWbcHhP2QVy/4SW+uvSrd9q392x7N3RxOR30yEymWoO+Uh1z4eAevP6f59KnII3bn1vKL99Y2zRNUNmvweMN63TNQMOKfUF/1Q77gr7HtK+nD77BXO3pK9UJZbmpvHT7eG4Y34sn5m9mxuyP2X0ourMjVHBujyExDLtmBZOV6qI8N5XVdgZ9b2j19AMVZ6fo7B2lOispwcm904bx0PRK1u6q4/KH5/NRlS7ospt/ymakDC/JsrWn7zXtm6cPvqC/69AxvN7oTTvWoK+6rGmVJbw26xxy0hL5ypOL+P07n0f1P5c6VYPbkBChgVyAYSVZbN9/zLbia+0dyAUoyU6m0WOiurBMg77q0voVZvDqHecwbWQx//Ovz5g5Z7GtMzy6M7fXS2KEBnIhYDB3R13EHqM17R3IhZNz9aM5mKtBX3V5aUkJPHhtJb/44jAWbqzl8ofns2zbAbub1e00erwR7ulnAvYN5nZoIDcn+gu0NOirbkFE+MpZvfjrf5yNwyFc8/hC5ny0WUs4RFFjBKdsAmSnJlKWm2LbYK63nYuzwJ5VuRr0VbcyvDSLN/5zAhP7F3DPP9Yy6y/LOHIiugWvuqtILs7ys3Mw1+ul3XWFMpNdZCYnsKU2etsmatBX3U5WqosnbhjNDyYP4q1Vu7jikQ/ZsFtr90Sab3FWZEPOsJIstu2vt2Uw19OB2TsAg4syWbszeuMQGvRVt+RwCP9xXl/+fMtZ1B1zc9X/LmDdLnsGALuLRk9kp2yCvYO5HZm9A74PqnW76po2mYk0DfqqWxvfN4/XZp1DWpKTm+YsZk+dLuSKlGikd+xameufCtze2TsAQ4szOeH2silKu2hp0FfdXnF2Cn/62hjqjjUy86nFmuOPELc3sgO5ADlpiZTmRH8w12NNCOhIT3+o9UG1Jkp1gzToK4XvP94j153B+t11/Ofzn0btq3Z30uiO7JRNPzsGcz2d6On3LUgjKcHBmiilpEJ6BURksohsEJEqEbkryO1JIvKidfsiEamwrh8rIsutnxUicmV4m69U+Jw/qJB7pw3j3Q17uecfa3Q6Z5g1eiM/kAsnB3P3R3ERntff0+9A0E9wOhhUlBm1CqFtvgIi4gQeBS4DhgAzRGRIs8NuBg4YY/oBDwK/ta5fDYw2xlQCk4HHRSQhXI1XKty+clYvvj6xD899vI0/zt9sd3O6FF9OP7LpHYCJ/QsA+Nfa6JXX9vf0O5LeAV9ef+3Ouqh0NEL52B0LVBljNhljGoAXgGnNjpkGPG1dfgW4UETEGFNvjPEnSJMB7TqpmPeDyYOYMrwnv3xzHW+u2mV3c7qMaEzZBN/K3PLcVN5YFb2g77WygR1J74BvALruuJvqA5FfpBXKK1ACbA/4vdq6LugxVpA/BOQBiMg4EVkDrAJuD/gQaCIit4nIEhFZsnfv3vY/C6XCyOEQfndNJWeUZ/PtF5fzqZZsCAtfGYbI9/RFhMtHFPFR1b6o1Vk6OZDbsftHs4REKEE/2NNo3mNv8RhjzCJjzFBgDPBDEUk+7UBjZhtjRhtjRhcUFITQJKUiK9nl5IkbRtMjM5lbn16iWzCGQaPHS2IUevoAlw8vwuM1zF0Tnd5+U3qngz39gT0zSHQ6WLH9YDibFVQor0A1UBbweymws6VjrJx9FrA/8ABjzDrgKDCso41VKpry0pOYM3MMHmOY+dRi20r2dgUer8FriNjOWc0NLc6kV14qb0QpPecfyO1oeicpwcmQ4kyWx0jQXwz0F5HeIpIITAdea3bMa8CN1uUvA/OMMca6TwKAiPQCBgJbwtJypaKgT0E6s786muoDx7jtmaWccHvsblJc8m9bGY30DlgpnuFFLNhYG5VZPJ0dyAWoLMtm1Y5DTeeKlDaDvpWDnwXMBdYBLxlj1ojIvSJyhXXYk0CeiFQB3wH80zrPBVaIyHLg78A3jDG6hZGKK2N753L/1SP4ZMt+vv/KSp3K2QFuK5BFK70DcPmI6KV4OjNP329kWRb1DR4+r4lsHaiQpk8aY94E3mx23d0Bl48DVwe537PAs51so1K2m1ZZQvWBY9w/dwPlual895KBdjcprjS6o9vTBxhSlElFXipvrNzFjLHlEX0sbydW5PpVluUAsGL7QQb1zAxLu4LRFblKhegb5/Xl2tFl/H5eFS8t3t72HVSTRmtOYzSmbPr5Z/Es3FRLbYS3I+zsQC5ARV4qmckJEc/ra9BXKkQiwi+uHMaE/vn86O8NmFy6AAAgAElEQVSreHX5DrubFDfcHl9QjMbirECXDy/G4zW8HeEUT2cHcsH3/hpZls3y7ZGdtqlBX6l2cDkd/OH6MxhdkcO3XljOnz7UVbuh8A/kRrOnDzC4KIM+BWm8viKys3j8pZo6k94BGFWew4bddRw61hiGVgWnQV+pdspIdjFn5lgmD+3Jva+v5b631+vgbhsarZ5+NAquBRIRpg4vYtHmWmoOR65s9sn0TufOc26/fLwGFm6M3HwXDfpKdUCyy8mj15/BjLHl/OG9jdz111VambMVTT39CG+iEszUkcV4Dby9OnIpnqb0Tqd7+tlkJCXw/meRq0ygQV+pDnI6hF9dOYxvXtCPF5ds5/bnPuV4o87jD+ZkTj/6IWdAjwz6F6bz+srIpXjCMZALvr/P2f3y+OCzfRH79qhBX6lOEBG+c8lAfnbFUN5Zv4cbnvwkovnYeNUQ5cVZzV0+oojFW/ZHbGe0cAzk+k0aUMiOg8fYuPdIp88VjAZ9pcLgxrMr+P2MUSzbfoBrH1+o2y424099RXNxVqCpI4owhohVTQ3HPH2/iQPyAXhvQ2RSPBr0lQqTqSOKeeprY9m+v54v/WEBmyLUU4tH/hW50R7I9etXmMGgnhkRS/H4h3M6m9MHKM1JpU9BGvM/j8xgrgZ9pcLo3P75/OW2szje6OHLjy1kZXXkC2jFg4amKZv2pHcArhxVwtKtB1iyZX/bB7fTyTIM4TnfOX3zWbxlf9MAeDhp0FcqzEaUZvPy7eNJcTmZMftjPqrSclN2DuT6fXV8Lwozkvj1W+GfYhvO9A7A+L551Dd4WFkd/oVaGvSVioA+Ben87RtnU5abys1PL+aTzeHvXcaTaFfZDCY1MYFvXzyApVsPMHfNnrCeO1yzd/zO6pMHRGa+vgZ9pSKkR2Yyz90yjuLsFG6as5hVEei1xQu7VuQ2d/WZpfQtSOO+t9eHNXXiCePsHYDctEQG9cxg4abasJwvkAZ9pSIoPz2JP98yjqwUFzc+9QlVES6bG6v8K3JdUdpEpSUJTgd3XTaYTfuO8mIYi+Z5w1BPv7nxffNYsuVA2Pdw0KCvVIQVZaXw51vG4RDh+j8uYvv+erubFHX+KZuuBPvSO34XDS7kzF45PP7BxqZg3VnhTu8AjO+Txwm3l+XbwjsZQIO+UlFQkZ/Gc7eM5Xijl688uYiabjaPvymnb3NPH3wL6r56Vi+27z/Gx2FKn4SrDEOgM3tZ9fXDPAPM/ldAqW5iUM9M5swcw97DJ/jKk4s4EIVt/GKFP71j1+Ks5iYP60lGcgIvLglPiqepymYYe/p56UkUZyWzakdd2M4JGvSViqpR5Tn88cbRbKmt52tPfcKRE267mxQVbq/9s3cCJbucfLGyhLdW7+ZQfefLZvgHcsP9mTasJIvVO8I7AUCDvlJRdnbffP5w3Rms3lnHLU8v7hZF2k6WVo6NoA9w7ZgyGtxeXl3R+c1w/GMD4UzvAAwvyWLzvqPUHQ9fPScN+krZ4KIhPfjdNSNZtHk/d/z504isvIwlJ0srx07IGVaSxZCizLDM4onEQC7AsNIsANaEMcUTO6+AUt3MtMoSfvHFYbyzvoZvv7i8S/f4Gz1enA4J2zz2cLl2TBlrdtZ1OoXiicBALvh6+gBrdoYvxaNBXykbXT+uFz+aMojXV+7iC7//MOz521jh9hhb6+605IuVJSQ4pNOF2LwR6unnpydRlJXMqjC+LzToK2Wz2yb25embxnLoWCNffPQjHn23qsvtwtXg8cZUascvK9XFmb1yOr1T1cmB3PB/sA0rydKgr1RXM2lAAXPvnMilw3py/9wNXPP4QrbWHrW7WWHj9hhcCbEZbs4bWMi6XXWd2gMhUgO5AIOLMtm872jY0n+x+Soo1Q3lpCXyyIxRPDS9ks9rjnDZQ/N5ftG2LrHputvrJSHG8vl+5w0sAOhUbz9SA7kAg3pmYAx8vic8+zNo0FcqhogI0ypLmHvnREaVZ/Ojv6/i5qeXUHM4vlfwNriN7cXWWjKoZwY9MpN4vxM7VVkzUsNae8dvYM8MADbsCU/dpth8FZTq5oqzU3j2pnH89AtD+KhqH5c++AELN4a/4mK0uL3emBzIBd8H7aQBBcz/fG+Hx1K8Yd5EJVCv3FQSExxs2B2eaZsa9JWKUQ6HMPOc3rzxzXPJS09i5pxP4jbwN3q8tm2VGIpJAwqpO+5m+faO1bnxRnAgN8HpoH9hOut3a09fqW6hX2EGL9x2FmU5qcyc8wkLIrCxRqQ1emI3vQO+bS6dDunwZuSRmqfvN7BHBp9pekep7iM/PYm/WIH/pjmL4y7wN3piN70DkJXi4ozybN77rKZD94/UPH2/gT0z2FN3goP1nS/Sp0FfqTjhD/zluVbgj6O9d90x3tMHmNi/gNU76jpUgK2pymaEevoD/IO5YUjxxParoJQ6RX56Es/fagX+p+Mn8Dd6YnfKpt/IsmygYyUP/OmdCMV8BllBPxx5fQ36SsUZf+DvlZvGTU8v5qM4CPy+9E5sh5thVp2b1R0I+l6vwSG+mUCR0DMzmfz0JJZtO9Dpc4X0KojIZBHZICJVInJXkNuTRORF6/ZFIlJhXX+xiCwVkVXWvxd0usVKKd/eu7eOo1duGjfHQeB3e2Oz9k6g3LRESrJTWN2BipYeYyKWzwffh8mYihyWbI1C0BcRJ/AocBkwBJghIkOaHXYzcMAY0w94EPitdf0+4AvGmOHAjcCznW6xUgo4NfDf9swSdhw8ZneTWtTgju0pm35DizM7VPTO19OP7Ifa6Ipcqg8cY/ehzi3UC+VVGAtUGWM2GWMagBeAac2OmQY8bV1+BbhQRMQYs8wYs9O6fg2QLCJJnWqxUqpJfnoSf7xxNF4DP/m/1TFbssHtNTGzVWJrhpVksWnfUQ63c9MSjzeyPX2A0daeuUu27u/UeUJ5FUqAwF0Gqq3rgh5jjHEDh4C8ZsdcBSwzxpzoWFOVUsGU5aby3UsGMG99TadLBEeKb3FWbKd34GT9+nW72jdg6jEmYjN3/IYUZ5LicrJkS+dSPKEE/WDPpHl3otVjRGQovpTP14M+gMhtIrJERJbs3du5EqdKdUczz+nNiNIsfvaPNWGZyx1u8TBlE2BoSSZAu1M8Xq+J+AYxLqeDyrLsqPT0q4GygN9LgZ0tHSMiCUAWsN/6vRT4O3CDMWZjsAcwxsw2xow2xowuKCho3zNQSuF0CL/50ggO1DfyqzfX2d2c08T64iy/woxkCjOS2h30Iz2Q63dmrxzW7qzjhLvjZZZDCfqLgf4i0ltEEoHpwGvNjnkN30AtwJeBecYYIyLZwBvAD40xH3W4lUqpNg0pzuTWCX14aUl1zK3Y9c3Tj/2ePvhSPO2dtunxRq4EQ6De+Wl4Dew40PFB+zZfBStHPwuYC6wDXjLGrBGRe0XkCuuwJ4E8EakCvgP4p3XOAvoBPxGR5dZPYYdbq5Rq1Z0X9adXXio/+tuqmNpzN17SOwBDS7KoqjnCsYbQ/35eryEaT68sNxWA7ZEM+gDGmDeNMQOMMX2NMb+0rrvbGPOadfm4MeZqY0w/Y8xYY8wm6/pfGGPSjDGVAT8dK26hlGpTssvJr64czpbaeh5+53O7m9OkIU7SOwDDijPxGljXjlLG0RjIBSi3gv62/fUdPkd8fPQqpUJ2Tr98vnxmKbM/2MS6XeGpwd5ZvsVZ8RFuhpdaK3PbkdePxkAuQGFGEokJDqo16CulAv14ymCyUlzc9deVTVv52cXrNXi8Ji6mbIKv5EGPzCQWbQp9lky0BnIdDqE0J0V7+kqpU+WkJXL3F4awovoQTy/YYmtbGr2+EpTx0tMXES4YVMj7n+2lwR3aTloeb3TSOwBlOalsP6BBXynVzBUjizlvYAEP/uszW+fuu60NZOMlpw9w4aAeHDnhZtHm0HYq85ropHfAl9ffvj/CA7lKqfgjItx12SCONLiZ/cEm29rRaBWbj5cpm+AbF0l2OXhnXWjzTqLa089N4dCxRg4da3/df9Cgr1SXNqhnJlNHFDNnwRb2HbGnAkqjv6efED/hJiXRybn9CvjX2j0h1TPyeIlqTx9gewfz+vHzKiilOuTOi/pzvNHDY+8FXRAfcf6evivGN1Fp7qLBhew4eCykjUu8Jjrz9AFKczToK6Va0bcgnS+dUcqzH29lT13nyvJ2xMmcfnyFmwsG+9aRvrNuT5vHeqM0Tx8CF2hp0FdKteBbF/bH4zU8+m5V1B+7wZ/Tj6OBXPDV4RlZls2/Qsjre6I0Tx98m7hnJCWw82DHPsA16CvVDZTlpnLNmDL+8sk2qjsx3a8j3NaUzXiop9/c+QMLWFl9kLo26utHs6cP0DMrucObqcTfq6CU6pBZ5/dDEB6ZF93evj+9Ew87ZzV3RnkOxsDq6tZX50azpw++oL+rg6m6+HsVlFIdUpydwnXjynl5aTVb9h2N2uPGa3oHYIRVkmF59cFWj/N6iW5PPzOZPdrTV0q15Rvn98XllKgWY/P39OMxvZOdmkjv/DRWbG896HuMIZrLEHpmJVNz+DhuT2grhgPF36uglOqwwoxkbhxfwd+X76Cqpn1bAnbUycVZ8dfTBxhZmsWK7SGkd6LY0++RmYzXwL4j7V9prUFfqW7m65P6kupy8uC/o9Pbb5qnH0eLswKNLMtmd93xVgdOvVEquOZXlJUMwK5D7S/HEJ+vglKqw3LTErnp3N68sXIXCzeGVlumM5pW5MZRGYZAI8uyAVjeSoonmmUYwNfTBzq07iI+XwWlVKd8fVJf+hakMev5TzvUW2wPd1NPPz7TO0OKMklwCCtaGcyN9uydkz19DfpKqRCkJyXw+FdHc8Lt5fbnPu3URtttabTq+cdTwbVAyS4ng4syWx3MjfY8/dy0RBKdDnZrT18pFap+hek8cPVIVmw/yD2vrYnY4zS6/fX047OnDzCyLIuV1YfwtrAhjccb3Zy+iFCYmdShBVoa9JXqxiYP68kd5/flL59s5y+fbIvIY7jjbBOVYEaWZnPkhJtN+44Evd1roldl06+og6ty4/dVUEqFxXcuHsjEAQX89NU1LNt2IOznb2hakRu/Pf1KazD3063BUzy+gdxotsg3mKvpHaVUuzkdwsPTK+mRlcR/PPcpew+Ht+6+fyA3Hhdn+fUtSCcn1cUnW4LvmxvtgVw42dMPpd5/oPh9FZRSYZOdmshjXzmTg8cauOP5T8O6mXrT4qw4DvoOhzCmIpdPNgcP+tEeyAXfQrsTbi91x93tul/8vgpKqbAaWpzFvdOG8cnm/fz10+qwnbcxDvfIDWZs71y27a8POsU12gO5AAUZSQDt/mamQV8p1eTqM0upLMvmgbkbqG9oXw+yJe44X5zld1afPICgvf1obozuV6hBXynVWSLCT6YOpubwCR5/PzybqTd6vDgk+rNbwm1wUSbpSQksChL0o70iFwJ6+u3c+1iDvlLqFGf2yuXy4UXM/mBTWLZXbPR643q6pp/TIYyuyAna09f0jlIqrv1g8iA8XsMDczd0+lyNbtMlgj7AuN55VNUcYV+z3rXXENUqm+DbNtHlFA36SqnOK89L5caze/HKp9Ws2dl6WeG2uL3euB/E9RvbOxeAJc2mbvp6+tFti4hQkJ6kQV8pFR6zLuhPdoqLX76xrt1zwQM1erxxPV0z0PCSLJJdDj7edGrQt2MgF3wpnprD7UvBdY1XQikVdlkpLr51YX8WbKxl3vqaDp+n0WPiemFWoMQEB0OKMlm/u+6U6+2Ypw++oK89faVU2Fx/Vi/65KfxqzfXNS2yai+3xxvXJRia61uQTlXNqXsM2zGQC1CQkXza+EJbNOgrpVrkcjq467JBbNx7lCfmd2wKZ6PHxO1WicH0K0xn35ETHKpvBMAYY8tALvh6+rVHG9q1V64GfaVUqy4e0oMpw3ty39sb+Puy9q/UbfR0jSmbfv0K0wGo2uuruOmvWGFPTz8JY2D/0dD3yg3plRCRySKyQUSqROSuILcniciL1u2LRKTCuj5PRN4VkSMi8kjIrVJKxQwR4XfXVDK+Tx7fe3kl89bvadf9u1rQ71vgC/obraDvr1NkS9BP983Vr2lHXr/NV0JEnMCjwGXAEGCGiAxpdtjNwAFjTD/gQeC31vXHgZ8A3wu5RUqpmJPscjL7hjMZUpTJfzz3KYtbqDYZjNtrusyUTYCy3FQSnQ421vh7+r6gb0N2p0OrckP5+B0LVBljNhljGoAXgGnNjpkGPG1dfgW4UETEGHPUGPMhvuCvlIpjGcku5swcQ0l2CjfNWcy6XXVt3wlocHedKZvg69H3zk87vadvQ9TvSP2dUF6JEmB7wO/V1nVBjzHGuIFDQF6ojRCR20RkiYgs2bt3b6h3U0pFWV56Es/eMo70pAS++uQnbK092uZ93N6uM2XTr19hOlVWT99j7Evv5KYlAnAgzDn9YM+k+UqNUI5pkTFmtjFmtDFmdEFBQah3U0rZoCQ7hWdvHkujx8uP/766zeO72pRNgL4FaWzbX88Jt6dp31w7Zu+kuJwA1DeEvrF9KEG/GigL+L0U2NnSMSKSAGQBoSf9lFJxpV9hBrdO6M2HVfuoqjnc6rENHkNCnJdVbq5vYTpeA1v21ds6kOtwCMkuB8cawxv0FwP9RaS3iCQC04HXmh3zGnCjdfnLwDzTmXXbSqmYN31sOYlOB08v2NrqcW6Pl8SErtXT90/b3Lj3SFN6x67S0amJCe3a+6DNoG/l6GcBc4F1wEvGmDUicq+IXGEd9iSQJyJVwHeApmmdIrIF+B3wNRGpDjLzRykVh/LTk5g6soi/flpN3fHGFo9r9Hi7XE+/T346IlBVcwSvtS7KjoFc8KV42pPeSQjlIGPMm8Cbza67O+DyceDqFu5bEXJrlFJx5WtnV/C3T3fwypJqbjq3d9BjGj1dp7SyX0qik5LslFN6+nY9xdREJ8fCnNNXSqmgRpRmM6o8m2cWbmka0GzOtzira6V3wF+D54itA7ngC/rhHshVSqkWfe3sCrbU1vP+58GnW/sWZ3W9UFOWm8LuQ8dtHcgF37cO7ekrpaLmsmFFFGQk8fSCLUFvb3R3vSmbALmpiRyob8BtJfXtCvqpiQnUN4ZxIFcppVqTmODg+nHlvLdhL5v3nb5Yq6vskdtcTloiXgMHrGqbdqV3UjS9o5SKtuvGleNyCs8s3HLabW5P16q945eT6lsNu88qgWBbT9+l6R2lVJQVZiQzZXgRryyp5uiJk6kGYwxub9dbnAW+nj7QtImJDuQqpbqVG8+u4PAJN6+vPLlgv9HjG+RMTOh6oSbX6unvPeKre2PfQG6C9vSVUtE3qiyb7FQXy7YdbLrOv8ViV9o5yy8nzQVA7RF/eseedqQmOmnQnbOUUtEmIgwtzmRtQMllt9XT74oDubkxlN5pj673SiilbDO0OIv1uw839fD9PdCuOJCb4nKSlOBgn+3pHQ36SimbDCnKpMHtbdpgxD+HvSttouInIuSmJTb19O2qvaM9faWUbYYWZwKwdqcvxdOV0zvgm7bpn7JpV5XNFFdIJdSadM1XQilliz4F6SS7HKyxgn5XTu+AL69/1Jo5Y9+KXO3pK6Vs4nQIg3pmsmbnIaAb9PStwVzQgVylVDc1pDiTtTvrMMZ06SmbALmprqbLdvX0k10a9JVSNhpanEndcTfVB441BX1XF1ycBc17+va0QXv6SilbDS3OAmDNzrqmFbmuLliGAU7O1Qc70zs6kKuUstHAHhk4BNbuPITbn97pogO5/qJroPP0lVLdVEqik74F6azdVUejt2sP5Ab29HX2jlKq2xpanOlL77i79pTNwJ6+Xekdl9PRrr+vBn2lVNgNLc5i16Hj1FgLl7SnH1kp7ZjB0zVfCaWUrYZYK3NXbPdV3OyqPf3swCmbNvX0oX2DuRr0lVJh5y/HsKLaH/S7ZqhJdjmbcup2TlBqT16/a74SSilbZacmUpKdwmd7DgNds+Canz+vb2t6R4O+Uspug4sysSbv4OqiK3LhZF7f3vSOBn2llM38KR7ouukdOLkq164qm+DbMjFUXfeVUErZKjDod9XFWXCy/o6tPX2dvaOUstsQ7elHjaZ3lFK2K8lOISvF1wvuykE/1xrItbOS6E3n9g752PZV6lFKqRD5N0r/eFOtrTNbIu3KM0rITHGRlmRfOB1WkhXysV3341cpZbsze+Wcsmq1KyrNSeXGsyvsbkbItKevlIqYO87vx4yx5XY3QwXQnr5SKmKSXU6Ks1PsboYKEFLQF5HJIrJBRKpE5K4gtyeJyIvW7YtEpCLgth9a128QkUvD13SllFLt1WbQFxEn8ChwGTAEmCEiQ5oddjNwwBjTD3gQ+K113yHAdGAoMBn4g3U+pZRSNgilpz8WqDLGbDLGNAAvANOaHTMNeNq6/ApwoYiIdf0LxpgTxpjNQJV1PqWUUjYIZSC3BNge8Hs1MK6lY4wxbhE5BORZ13/c7L4lzR9ARG4DbrN+PSIiG1ppTxZwqJXb84F9rdwez9p67vH8+OE6d0fP0977tef4UI4N5Rh9b8ff40fzfd0rlBOFEvSDTbA1IR4Tyn0xxswGZofQFkRktjHmtlZuX2KMGR3KueJNW889nh8/XOfu6Hnae7/2HB/KsSEeo+/tOHt8u9/XwYSS3qkGygJ+LwV2tnSMiCTg+1TaH+J92+sfnbx/PLP7uUfy8cN17o6ep733a8/xoRxr92trN7uff6Qe3+739WnEmNM63qce4AvinwEXAjuAxcB1xpg1AcfcAQw3xtwuItOBLxljrhGRocDz+PL4xcA7QH9jjCdcTyBIe7tsb0h1b/reVuHQZnrHytHPAuYCTuBPxpg1InIvsMQY8xrwJPCsiFTh6+FPt+67RkReAtYCbuCOSAZ8S0hpIqXikL63Vae12dNXSinVdeiKXKWU6kY06CulVDcStaAvIiki8r5/Ra6I3Cgin1s/N7Zwn/tFZL2IrBSRv4tIdhuPUSYi74rIOhFZIyLfCrjtHhHZISLLrZ8pAbcFLRXRUvkJEXlBRPp35u+hup54e4+LSKKIfGBN1lDdhTEmKj/AHcC3rMu5wCbr3xzrck6Q+1wCJFiXfwv8to3HKALOsC5n4Jt1NMT6/R7ge0HuMwRYASQBvYGN+AasndblPkCidYz/XJOAJ6L1t9Of+PiJt/e4ddtPgevt/tvpT/R+opneuR541bp8KfAvY8x+Y8wB4F/4avOcwhjzT2OM2/r1Y3zz/FtkjNlljPnUunwYWEeQFcDNtFQqorXyE/OBi7SHpJqJt/c4wP9Z7VbdRFSCvogkAn2MMVusq4KVdmjrjXsT8FY7HrMCGAUsCrh6lvU1+k8iktNGW1psozHGi+8/zshQ26O6tjh9jwOsBsaE+pgq/kWrp58PHAz4PaTyDE0Hi/wY3zz/P4fyYCKSDvwVuNMYU2dd/b9AX6AS2AX8TxttaauNNfgWnCkF8fkex/jWzTSISEYoj6viX7SC/jEgOeD3kMszWANgU/HlHdtcVCAiLnz/Gf5sjPmb/3pjzB5jjMfqpT/Bya+3LbWlrTYmW89LKYjP97hfEnC8rcdVXUS0Bg/wfb1Mti7nApvxDXDlWJdzg9xnMr7VvAXNri8B3glyvADPAP8vyG1FAZe/jS/HCb5a/4GDXJvwDeImWJd7c3Igd2jAOVYFnlN/9Cfe3uPWbXnAOrv/dvoTvZ9oDuT+EzgXwBizH/g5vjo+i4F7resQkT+KiL++yCP4Zij8y5qC9ph1fRG+r8LNnQN8FbggyLS1+0RklYisBM7H958C46sh5C8V8TZWqQjjG1zzl59YB7xkHYuI9ACOGWN2heUvo7qKuHqPW/c5H3gzPE9fxYOolWEQkVHAd4wxXw3DuWYB24yv7k/Uici3gTpjzJN2PL6KTfH4HheRvwE/NMa0toeF6kKiNuXQGLPMWlTiNJ0sumaMeSRc7eqgg8CzNrdBxZh4e49bM47+TwN+96IF15RSqhvR2jtKKdWNaNBXSqluRIO+Ukp1Ixr0lVKqG9Ggr+KWVUr4exF+jPf8c+pF5M22Sh9Hi4jMEZEv290OFX+0SqRSITLGTGn7qPYJx/ROpdpDe/oq5ojIDValyBUi8qyI9BKRd6zr3hGR8hDP800RWWvd7wXrunQRecq/clVErrKu/18RWWJtTPKzFs63RUTyRaTC2sTkCev4f4pIinXMGOu8C60NUlYHOc951nz+5/GV80BE/k9Ellrnuy3g2CMi8kvrb/GxtRq8+fl+bvX89f+zapO+SVRMEZGhwI+BC4wxI4Fv4StV8IwxZgS+KpQPh3i6u4BR1v1ut677CXDIGDPcun6edf2PjTGjgRHAJBEZ0ca5+wOPGmOG4lusd5V1/VPA7caY8UBrPfix1mMOsX6/yRhzJjAa+KaI5FnXpwEfW3+LD4BbA08iIvcBhcBM4yu0plSrNOirWHMB8IoxZh801bAZDzxv3f4sVn2bEKwE/iwiX+FkHZuLgEf9BxjfBicA14jIp8AyfAXKhtC6zcaY5dblpUCFle/PMMYssK5/PvhdAfjE+DY08fumiKzAt5FKGb4PFYAG4PXAxwm4z0+AbGPM142uslQh0qCvYo3QSt15S6gB7nJ8Af5MYKm109lp5xeR3sD3gAut3v8bnFomOZgTAZc9+MbHgtWtb8nRgMc/D9+H0XirR78s4PEbAwK6/3H8FgNnikhuOx5XdXMa9FWseQdfrzsPwApoC4Dp1u3XAx+2dRIrv11mjHkX+D6QDaTjq4Q5K+C4HCATXxA+ZOXML+tIw61vDYdF5CzrKn+bEZESEXmnhbtmAQeMMfUiMgg4q4Xjmnsb+A3whm6CokKls3dUTDHGrBGRXwLvi4gHX6/3m8CfROS/gL3AzBBO5QSeE5EsfD3wB40xB0XkF8Cj1gCrB/iZMeZvIrIMWIOv1tUPKGoAAACdSURBVPxHnXgKNwNPiMhR4D3gkHV9S6WSwRe8b7dKIm/Al+IJiTHmZSvgvyYiU4wxurGPapUWXFMqjEQk3RhzxLp8F76NTb5ldzlwpfw06CsVRiJyLfBDfN+itwJfM8bstbdVSp2kQV/FPRF5FN+OUoEeMsY8ZUd7lIplGvSVUqob0dk7SinVjWjQV0qpbkSDvlJKdSMa9JVSqhv5/9Bqg7ysAM7cAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "report_results('rank', scores);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'YaMus': {'PureSVDs': {'col_scaling': 0.4, 'rank': 3000}}}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### saving data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(experiment_name, config=config, tuning=scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## cross-validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:0; max-width:15ex; vertical-align:middle; text-align:right\"></span>\n",
       "<progress style=\"width:60ex\" max=\"1\" value=\"1\" class=\"Progress-main\"/></progress>\n",
       "<span class=\"Progress-label\"><strong>100%</strong></span>\n",
       "<span class=\"Iteration-label\">1/1</span>\n",
       "<span class=\"Time-label\">[09:58:36<09:58:36, 35916.00s/it]</span></div>"
      ],
      "text/plain": [
       "\u001b[A\u001b[A\u001b[2K\r",
       " [████████████████████████████████████████████████████████████] 1/1 [09:58:36<09:58:36, 35916.00s/it]\u001b[B"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:15ex; max-width:15ex; vertical-align:middle; text-align:right\">YaMus folds</span>\n",
       "<progress style=\"width:45ex\" max=\"5\" value=\"5\" class=\"Progress-main\"/></progress>\n",
       "<span class=\"Progress-label\"><strong>100%</strong></span>\n",
       "<span class=\"Iteration-label\">5/5</span>\n",
       "<span class=\"Time-label\">[09:58:36<02:00:29, 7183.20s/it]</span></div>"
      ],
      "text/plain": [
       "\u001b[A\u001b[2K\r",
       "    YaMus folds [█████████████████████████████████████████████] 5/5 [09:58:36<02:00:29, 7183.20s/it]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "result = {}\n",
    "\n",
    "for label in track(data_labels):\n",
    "    models = prepare_recommender_models(label, data_models, config)\n",
    "    result[label] = ee.run_cv_experiment(models,\n",
    "                                         fold_experiment=ee.topk_test,\n",
    "                                         topk_list=topk_values,\n",
    "                                         ignore_feedback=True,\n",
    "                                         iterator=lambda x: track(x, label=f'{label} folds'))\n",
    "    save_cv_training_time(experiment_name, models, label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEWCAYAAACKSkfIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGMRJREFUeJzt3X+QVOWd7/H3hxkE8fojDuhdQZ1RuQrCMLiAbolgSUS8ekVXLYHcklgY3QrcNdkkXsxuUIlldOO9Vii9ZUhpyRoNGNQNFTFWlESiZdDh8iMiUhlYlFldweGHGkQd+e4ffYC27XF6hmaa4fm8qqbmnOc8p/t7Ts185szTp59WRGBmZmnoUekCzMys6zj0zcwS4tA3M0uIQ9/MLCEOfTOzhDj0zcwS4tA3M0uIQ98OaZIelfRQQdtYSS2S/upL9quWFJLellSV135Ytm/rgazb7EBx6Nuh7u+B/y7pQgBJvYGfAd+JiHdK2P8DYHze+qXAe2Wv0qyLOPTtkBYRLcD/AuZKOgK4FVgfEQ9L+htJf5S0XdI7kuZI6lnwEI8A1+atXwv8S34HSc2Szs9bv0PSw9lyH0mPZf8dbJf0iqS+5T9Ss9I49O2QFxG/BJYDvwBuAG7MNrUCNwF9gXOBCXnb9ngSuEDSUZJqgHOAX3fg6a8D+gADgBrgm8Cuzh2J2f6rrnQBZl1kOrAe+MeIeAsgIl7N275B0lxgLHBfXvtO4BngauBw4Cng4w4876fk/qicFhF/Aho7fQRmZeDQtyRExLuS3gPW7GmTdAbwf4C/Jnc1Xg0sK7L7v5AbFjqc3H8GHfEwcALwuKSjyA0X/VNE+IVgqwgP71jKfgq8Ru4q/ChgFqAi/X4HnAwcExEvF9n+F3J/NPb4r3sWIuKTiLgtIgYBo4ErgK+VqX6zDnPoW8qOBHYAf5E0iC+O5wMQufnHLwUub+NxVgKTsts8RwF/u2eDpAskDZHUA3if3HDPZ2U8BrMOcehbyr4DTCV3W+ZPgQVtdYyI1yLi9TY2/yNwBrAd+AHwWN62E8i9GPw+uaGl58i9oGxWEfKHqJiZpcNX+mZmCXHom5klxKFvZpYQh76ZWUIOujdn9e3bN2praytdhplZt7J8+fL3IqJfe/0OutCvra2lsdHvVDcz6whJb5bSz8M7ZmYJceibmSXEoW9mlpCDbky/mE8//ZTm5mZ27fI05OXQu3dvBgwYQM+ehZ8XYmaHum4R+s3NzRx55JHU1tYiFZsE0UoVEbS0tNDc3ExdXV2lyzGzLtYthnd27dpFTU2NA78MJFFTU+P/mswS1S1CH3Dgl5HPpVm6uk3om5nZ/nPoH2C1tbW89957+93HzKwcusULuWZmB0rtzKcrXQIb77qky57LV/pFbNy4kTPOOIPrr7+eIUOG8LWvfY3nnnuOc889l4EDB/LKK6+wdetWLr/8curr6znnnHNYvXo1AC0tLYwfP57hw4dz4403kv8hNT//+c8ZNWoUDQ0N3HjjjXz2mT81z8y6lkO/DU1NTdx0002sXr2aN954g8cee4wXX3yRe+65hzvvvJNbb72V4cOHs3r1au68806uvfZaAG6//XZGjx7NihUruOyyy3jrrbcAWLt2LQsWLOCll15i5cqVVFVV8eijj1byEM0sQR7eaUNdXR1Dhw4F4Mwzz2TcuHFIYujQoWzcuJE333yTJ554AoALLriAlpYWduzYwdKlS3nyyScBuOSSS/jKV74CwPPPP8/y5csZOXIkAB999BHHHXdcBY7MzFLm0G9Dr1699i736NFj73qPHj1obW2luvqLp27PrZDFbomMCKZOncqPfvSjA1SxmVn7PLzTSWPGjNk7PPP73/+evn37ctRRR32u/ZlnnmHbtm0AjBs3joULF7J582YAtm7dyptvljQTqplZ2fhKv5Nuu+02rrvuOurr6+nTpw/z5s0D4NZbb2Xy5MmcddZZjB07lpNOOgmAwYMHc8cddzB+/Hh2795Nz549uf/++zn55JMreRiWqNTuWLF9lH93ycFgxIgRUfghKmvXrmXQoEEVqujQ5HOaNof+PofKuZC0PCJGtNfPwztmZglx6JuZJcShb2aWEIe+mVlCHPpmZgkpKfQlTZC0TlKTpJlFtveStCDbvkxSbdZeK+kjSSuzrwfKW76ZmXVEu/fpS6oC7gcuBJqBVyUtiojX87pNA7ZFxGmSJgF3A9dk29ZHREM5iy73LVal3C5VVVXF0KFDaW1tZdCgQcybN48+ffrs93O/++67TJs2jU2bNvHpp59SW1vL4sWLqaur4ze/+Q2nn3763r7f+ta3OOGEExg1ahQTJ07klFNOYefOnRx//PHcfPPNXHrppftdj5kd2kq50h8FNEXEhoj4BJgPTCzoMxGYly0vBMbpEPt4psMPP5yVK1fy2muvcdhhh/HAA6X/09La2trmtlmzZnHhhReyatUqXn/9de666y4AJk2axPz58/f22717NwsXLuSaa3J/S8877zxWrFjBunXrmDNnDjNmzOD555/v5NGZWSpKCf3+wKa89easrWifiGgFdgA12bY6SSskvSDpvGJPIOkGSY2SGrds2dKhA6iE8847j6amJjZu3MiQIUP2tt9zzz3cdtttAJx//vl8//vfZ+zYsfzkJz9hy5YtXHnllYwcOZKRI0fy0ksvAfDOO+8wYMCAvY9RX18PwOTJkz8X+kuXLqW2trboO3gbGhqYNWsW9913HwC//OUvGTJkCMOGDWPMmDFlP34z675KmYah2BV74dt42+rzDnBSRLRI+mvgXyWdGRHvf65jxFxgLuTekVtCTRXT2trKM888w4QJE9rtu337dl544QUApkyZwre//W1Gjx7NW2+9xUUXXcTatWuZPn0611xzDffddx9f/epXue666zjhhBOor6+nR48erFq1imHDhjF//nwmT57c5nOdddZZ/PjHPwZg9uzZPPvss/Tv35/t27eX58DN7JBQypV+M3Bi3voA4O22+kiqBo4GtkbExxHRAhARy4H1wH/b36Ir4aOPPqKhoYERI0Zw0kknMW3atHb32TMUA/Dcc88xY8YMGhoauOyyy3j//ff54IMPuOiii9iwYQPf+MY3eOONNxg+fDh7/tvZc7Xf2trKr371K66++uo2nyt/Oo1zzz2Xr3/96/zsZz/zB7WY2eeUcqX/KjBQUh3w78AkYEpBn0XAVOBl4CpgSUSEpH7kwv8zSacAA4ENZau+C+0Z089XXV3N7t27967v2rXrc9uPOOKIvcu7d+/m5Zdf5vDDD//CYx977LFMmTKFKVOmcOmll7J06VKuvPJKJk+ezPjx4xk7diz19fVfOv/+ihUr9s6l88ADD7Bs2TKefvppGhoaWLlyJTU1NW3um4pDZY4Vs/3R7pV+NkY/A3gWWAs8HhFrJM2WdFnW7UGgRlIT8A/Ants6xwCrJa0i9wLv30XE1nIfRKUcf/zxbN68mZaWFj7++GN+/etft9l3/Pjxe8fcgb1/QJYsWcLOnTsB+OCDD1i/fv3emTlPPfVUampqmDlz5pcO7axevZof/vCHTJ8+HYD169dz9tlnM3v2bPr27cumTZva3NfM0lLS1MoRsRhYXNA2K295F/CFsYeIeAJ4Yj9r/IKD5WqpZ8+ezJo1i7PPPpu6ujrOOOOMNvvOmTOH6dOnU19fT2trK2PGjOGBBx5g+fLlzJgxY+9/Dddff/3eT9eC3BDPLbfcwhVXXPG5x/vDH/7A8OHD2blzJ8cddxxz5sxh3LhxAHzve9/jz3/+MxHBuHHjGDZs2IE5AWbW7Xhq5USleE49vLOPz8U+h8q58NTKZmb2BQ59M7OEdJvQP9iGobozn0uzdHWL0O/duzctLS0OqzKICFpaWujdu3elSzGzCugWH4w+YMAAmpub6Q5TNHQHvXv3/tzUD2aWjm4R+j179qSurq7SZZiZdXvdYnjHzMzKw6FvZpYQh76ZWUIc+mZmCXHom5klxKFvZpYQh76ZWUIc+mZmCXHom5klxKFvZpYQh76ZWUIc+mZmCXHom5klxKFvZpYQh76ZWUIc+mZmCXHom5klxKFvZpYQh76ZWUIc+mZmCXHom5klxKFvZpaQkkJf0gRJ6yQ1SZpZZHsvSQuy7csk1RZsP0nSh5K+W56yzcysM9oNfUlVwP3AxcBgYLKkwQXdpgHbIuI04F7g7oLt9wLP7H+5Zma2P0q50h8FNEXEhoj4BJgPTCzoMxGYly0vBMZJEoCky4ENwJrylGxmZp1VSuj3BzblrTdnbUX7REQrsAOokXQE8L+B27/sCSTdIKlRUuOWLVtKrd3MzDqolNBXkbYosc/twL0R8eGXPUFEzI2IERExol+/fiWUZGZmnVFdQp9m4MS89QHA2230aZZUDRwNbAXOBq6S9M/AMcBuSbsi4r79rtzMzDqslNB/FRgoqQ74d2ASMKWgzyJgKvAycBWwJCICOG9PB0m3AR868LtW7cynK10CABvvuqTSJZgZJYR+RLRKmgE8C1QBD0XEGkmzgcaIWAQ8CDwiqYncFf6kA1m0mZl1TilX+kTEYmBxQdusvOVdwNXtPMZtnajPzMzKyO/INTNLiEPfzCwhDn0zs4Q49M3MEuLQNzNLiEPfzCwhDn0zs4Q49M3MEuLQNzNLiEPfzCwhDn0zs4Q49M3MEuLQNzNLiEPfzCwhDn0zs4Q49M3MEuLQNzNLiEPfzCwhDn0zs4Q49M3MEuLQNzNLiEPfzCwhDn0zs4Q49M3MEuLQNzNLiEPfzCwhDn0zs4Q49M3MEuLQNzNLSEmhL2mCpHWSmiTNLLK9l6QF2fZlkmqz9lGSVmZfqyRdUd7yzcysI9oNfUlVwP3AxcBgYLKkwQXdpgHbIuI04F7g7qz9NWBERDQAE4CfSqouV/FmZtYxpVzpjwKaImJDRHwCzAcmFvSZCMzLlhcC4yQpInZGRGvW3huIchRtZmadU0ro9wc25a03Z21F+2QhvwOoAZB0tqQ1wJ+Av8v7I7CXpBskNUpq3LJlS8ePwszMSlJK6KtIW+EVe5t9ImJZRJwJjARukdT7Cx0j5kbEiIgY0a9fvxJKMjOzzigl9JuBE/PWBwBvt9UnG7M/Gtia3yEi1gJ/AYZ0tlgzM9s/pYT+q8BASXWSDgMmAYsK+iwCpmbLVwFLIiKyfaoBJJ0MnA5sLEvlZmbWYe3eSRMRrZJmAM8CVcBDEbFG0mygMSIWAQ8Cj0hqIneFPynbfTQwU9KnwG7gmxHx3oE4EDMza19Jt09GxGJgcUHbrLzlXcDVRfZ7BHhkP2s0M7My8TtyzcwS4tA3M0uIQ9/MLCEOfTOzhDj0zcwS4tA3M0uIQ9/MLCEOfTOzhDj0zcwS4tA3M0uIQ9/MLCEOfTOzhBySn1dbO/PpSpcAwMa7Lql0CWZmn+MrfTOzhDj0zcwS4tA3M0uIQ9/MLCEOfTOzhDj0zcwS4tA3M0uIQ9/MLCEOfTOzhDj0zcwS4tA3M0uIQ9/MLCEOfTOzhDj0zcwS4tA3M0uIQ9/MLCElhb6kCZLWSWqSNLPI9l6SFmTbl0mqzdovlLRc0p+y7xeUt3wzM+uIdkNfUhVwP3AxMBiYLGlwQbdpwLaIOA24F7g7a38P+B8RMRSYCjxSrsLNzKzjSrnSHwU0RcSGiPgEmA9MLOgzEZiXLS8ExklSRKyIiLez9jVAb0m9ylG4mZl1XCmh3x/YlLfenLUV7RMRrcAOoKagz5XAioj4uPAJJN0gqVFS45YtW0qt3czMOqiU0FeRtuhIH0lnkhvyubHYE0TE3IgYEREj+vXrV0JJZmbWGaWEfjNwYt76AODttvpIqgaOBrZm6wOAp4BrI2L9/hZsZmadV0rovwoMlFQn6TBgErCooM8ici/UAlwFLImIkHQM8DRwS0S8VK6izcysc9oN/WyMfgbwLLAWeDwi1kiaLemyrNuDQI2kJuAfgD23dc4ATgN+IGll9nVc2Y/CzMxKUl1Kp4hYDCwuaJuVt7wLuLrIfncAd+xnjWZmViZ+R66ZWUIc+mZmCXHom5klxKFvZpYQh76ZWUIc+mZmCXHom5klxKFvZpYQh76ZWUIc+mZmCXHom5klxKFvZpYQh76ZWUIc+mZmCXHom5klxKFvZpYQh76ZWUIc+mZmCXHom5klxKFvZpYQh76ZWUIc+mZmCXHom5klxKFvZpYQh76ZWUIc+mZmCXHom5klxKFvZpYQh76ZWUJKCn1JEyStk9QkaWaR7b0kLci2L5NUm7XXSPqdpA8l3Vfe0s3MrKPaDX1JVcD9wMXAYGCypMEF3aYB2yLiNOBe4O6sfRfwA+C7ZavYzMw6rZQr/VFAU0RsiIhPgPnAxII+E4F52fJCYJwkRcRfIuJFcuFvZmYVVkro9wc25a03Z21F+0REK7ADqClHgWZmVj6lhL6KtEUn+rT9BNINkholNW7ZsqXU3czMrINKCf1m4MS89QHA2231kVQNHA1sLbWIiJgbESMiYkS/fv1K3c3MzDqolNB/FRgoqU7SYcAkYFFBn0XA1Gz5KmBJRJR8pW9mZl2jur0OEdEqaQbwLFAFPBQRayTNBhojYhHwIPCIpCZyV/iT9uwvaSNwFHCYpMuB8RHxevkPxczM2tNu6ANExGJgcUHbrLzlXcDVbexbux/1mZlZGfkduWZmCXHom5klxKFvZpYQh76ZWUIc+mZmCXHom5klxKFvZpYQh76ZWUIc+mZmCXHom5klxKFvZpYQh76ZWUIc+mZmCXHom5klxKFvZpYQh76ZWUIc+mZmCXHom5klxKFvZpYQh76ZWUIc+mZmCXHom5klxKFvZpYQh76ZWUIc+mZmCXHom5klxKFvZpYQh76ZWUIc+mZmCXHom5klpKTQlzRB0jpJTZJmFtneS9KCbPsySbV5227J2tdJuqh8pZuZWUe1G/qSqoD7gYuBwcBkSYMLuk0DtkXEacC9wN3ZvoOBScCZwATg/2WPZ2ZmFVDKlf4ooCkiNkTEJ8B8YGJBn4nAvGx5ITBOkrL2+RHxcUT8G9CUPZ6ZmVVAdQl9+gOb8tabgbPb6hMRrZJ2ADVZ+x8L9u1f+ASSbgBuyFY/lLSupOoPrL7Ae/vzALq7TJVUns/FPvt1Lg6h8wA+F/kOhnNxcimdSgl9FWmLEvuUsi8RMReYW0ItXUZSY0SMqHQdBwOfi318LvbxudinO52LUoZ3moET89YHAG+31UdSNXA0sLXEfc3MrIuUEvqvAgMl1Uk6jNwLs4sK+iwCpmbLVwFLIiKy9knZ3T11wEDglfKUbmZmHdXu8E42Rj8DeBaoAh6KiDWSZgONEbEIeBB4RFITuSv8Sdm+ayQ9DrwOtALTI+KzA3Qs5XZQDTdVmM/FPj4X+/hc7NNtzoVyF+RmZpYCvyPXzCwhDn0zs4Q49AtIekjSZkmvVbqWSpLUW9IrklZJWiPp9krX1NWK/SxIOlbSbyX9Ofv+lUrW2BUknSjpd5LWZj8LN2XtKZ6Lor8X2Y0uy7JzsSC76eWg5ND/oofJTRmRuo+BCyJiGNAATJB0ToVr6moP88WfhZnA8xExEHg+Wz/UtQLfiYhBwDnA9GyKlRTPRVu/F3cD92bnYhu5qWkOSg79AhGxlNwdSEmLnA+z1Z7ZV1Kv+rfxs5A/5cg84PIuLaoCIuKdiPj/2fIHwFpy76xP8Vy09XtxAbkpaOAgPxcOfWuTpCpJK4HNwG8jYlmlazoIHB8R70AuDIHjKlxPl8pm0B0OLCPRc1H4ewGsB7ZHRGvWpeh0MwcLh761KSI+i4gGcu+kHiVpSKVrssqR9F+AJ4BvRcT7la6nUgp/L4BBxbp1bVWlc+hbuyJiO/B7/FoHwLuS/gog+765wvV0CUk9yQX+oxHxZNac5LnYI+/34hzgmGwKGjjIp5tx6FtRkvpJOiZbPhz4KvBGZas6KORPOTIV+FUFa+kS2TTpDwJrI+L/5m1K8VwU+71YC/yO3BQ0cJCfC78jt4CkXwDnk5sq9V3g1oh4sKJFVYCkenIvSFWRuzh4PCJmV7aqrlXsZwH4V+Bx4CTgLeDqiDikX/iXNBr4A/AnYHfW/H1y4/qpnYuivxeSTiH3WSPHAiuA/xkRH1eu0rY59M3MEuLhHTOzhDj0zcwS4tA3M0uIQ9/MLCEOfTOzhDj0LVmSjpH0zUrXYdaVHPqWsmMAh74lpd3PyDU7hN0FnJpNnvXbrO1icvOm3BERCySdD8wGWoDTgaXANyNid/4DSfo6cBnQBzgVeCoibu6KgzDrCF/pW8pmAuuzybP+SG5+9GHk3lr/4z3zypCbVOs7wFBygf63bTxeA3BN1u8aSScewNrNOsWhb5YzGvhFNoPiu8ALwMhs2ysRsSEiPgN+kfUt5vmI2BERu4DXgZMPeNVmHeTQN8vRl2wrnKskJF0haWX2NSJrz59r5TM8fGoHIYe+pewD4MhseSm5IZkqSf2AMcAr2bZR2Weg9iA3fPNiRDwVEQ3ZV2PXl27WOQ59S1ZEtAAvZR98/jfAamAVsAS4OSL+I+v6MrkXfV8D/g14qgLlmpWFZ9k0+xLZ3TvfjYhLK12LWTn4St/MLCG+0jczS4iv9M3MEuLQNzNLiEPfzCwhDn0zs4Q49M3MEvKf3/pVzfuqWCAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "report_results('topn', result, target_metric);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### saving data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(experiment_name, cv=result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cold start"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "import gc\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_cold_start_recommender_models(data_label, data_models, config):\n",
    "    data_model = data_models[data_label]\n",
    "    svd = ScaledSVDItemColdStart(data_model, item_features=meta_dict[data_label])\n",
    "    svd.max_test_workers = max_test_workers\n",
    "    models = [svd]\n",
    "    apply_config(models, config, data_label)\n",
    "    return models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "config_cold = {}\n",
    "scores_cold = {}\n",
    "data_models_cold = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:0; max-width:15ex; vertical-align:middle; text-align:right\"></span>\n",
       "<progress style=\"width:60ex\" max=\"1\" value=\"1\" class=\"Progress-main\"/></progress>\n",
       "<span class=\"Progress-label\"><strong>100%</strong></span>\n",
       "<span class=\"Iteration-label\">1/1</span>\n",
       "<span class=\"Time-label\">[07:09:28<07:09:28, 25767.85s/it]</span></div>"
      ],
      "text/plain": [
       "\u001b[A\u001b[A\u001b[2K\r",
       " [████████████████████████████████████████████████████████████] 1/1 [07:09:28<07:09:28, 25767.85s/it]\u001b[B"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:15ex; max-width:15ex; vertical-align:middle; text-align:right\">YaMus</span>\n",
       "<progress style=\"width:45ex\" max=\"57\" value=\"57\" class=\"Progress-main\"/></progress>\n",
       "<span class=\"Progress-label\"><strong>100%</strong></span>\n",
       "<span class=\"Iteration-label\">57/57</span>\n",
       "<span class=\"Time-label\">[07:09:03<01:18, 451.62s/it]</span></div>"
      ],
      "text/plain": [
       "\u001b[A\u001b[2K\r",
       "          YaMus [█████████████████████████████████████████████] 57/57 [07:09:03<01:18, 451.62s/it]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for label in track(data_labels):\n",
    "    data_models_cold[label] = prepare_cold_start_data_model(label, *all_data, seed)\n",
    "    model = ScaledSVDItemColdStart(data_models_cold[label],\n",
    "                                   item_features=meta_dict[label])\n",
    "    model.use_raw_features = True\n",
    "    config_cold[label], scores_cold[label] = fine_tune_scaledsvd(model,\n",
    "                                                                 svd_ranks[label],\n",
    "                                                                 scaling_params,\n",
    "                                                                 label, record_time=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/py36/lib/python3.6/site-packages/pandas/plotting/_core.py:1001: UserWarning: Attempted to set non-positive left xlim on a log-scaled axis.\n",
      "Invalid limit will be ignored.\n",
      "  ax.set_xlim(left, right)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEXCAYAAACtTzM+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmYXGW16P/vqqqunjs9pDMPnZAQyIxEBhEQIhAwEhSRBA6ghh+iCXqc7g+uh0HQc9SjVw8XjoiCIooROSJRwCjzHNKBjMRAZyDpJCSdudNTdVWt+8fe1al0qtNV3TVnfZ6nnuza9e53v1W7U6vecYuqYowxxngyXQBjjDHZwQKCMcYYwAKCMcYYlwUEY4wxgAUEY4wxLgsIxhhjAAsIxhhjXBYQzHFLRH4nIg9223euiOwRkaHHOM4nIioi20XEG7Xf7x4bTGW5jUkVCwjmePYV4BIRuQBARIqAXwDfUNUdcRzfDFwY9Xw2sDvppTQmTSwgmOOWqu4BbgLuF5FS4HZgg6r+WkTOFJE3RGS/iOwQkbtFpKBbFg8D10Y9vxb4TXQCEWkUkY9FPf+uiPza3S4RkUfcWsV+EXlTRAYm/50aEx8LCOa4pqp/BJYDvwduAL7ovhQEvgoMBM4CZkW9FvEn4HwRqRCRGuAM4K8JnP7zQAkwAqgBvgy09+2dGNN/vkwXwJgssADYAHxbVbcAqOqyqNc3isj9wLnAPVH7W4GngSuAYuBxoCOB83biBJxxqroaqO/zOzAmCSwgmOOequ4Ukd3A2sg+ETkJ+DFwKs6veB+wNMbhv8FpairGqVEk4tfAMOBREanAaYL6N1W1TmmTEdZkZExsPwfW4Px6rwBuAyRGuueB0UClqr4e4/UWnIASMSSyoaoBVb1DVU8GPgp8Crg6SeU3JmEWEIyJrRw4ALSIyMkc3X8AgDrrx88GLushnxXAXHeo6mnApyMviMj5IjJZRDzAQZwmpFAS34MxCbGAYExs3wCuwxla+nPgDz0lVNU1qvpODy9/GzgJ2A/cCjwS9downI7pgzjNVc/gdG4bkxFiN8gxxhgDVkMwxhjjsoBgjDEGsIBgjDHGZQHBGGMMkGMT0wYOHKh1dXWZLoYxxuSU5cuX71bV2t7S5VRAqKuro77eZvcbY0wiROT9eNJZk5ExxhjAAoIxxhiXBQRjjDGABQRjjDEuCwjGGGMACwjGGGNcOTXs1OQOVSUYVkJh99+QEgyHDz/v+jdMMKwEQ1HPQ4dfBxhYVsjgikKqS/2IxLolgTEmGSwgmLgEQ2F2HGhn695WtriPrfva2LK3lW372ujoDEV90YcJp2ARXb/XQ215IUMGFDG4opDBFUUMrihiSEURgyoKGeI+Ly20P2tj+sL+5xjA+UW/r7XT+aJ3v/Ab9x3+8t++v51Q1Le8zyMMrypmZFUJHz95EKWFPnwewesR918PPm/086j9kefeHvZHvR5W2N3cwQcH29l5sIOdB9vZebCdf37QzEvv7uZQx9F3mywv9DkBYkARg8uLGDygiIqiAhQlstp7OKwooAphdbZx/w2rk04BrwhnjK3hjLHV+LzWwmryW1wBQURmAf8FeIFfqur3u71eiHNv2VOBPcCVqrrZvUPU/ZFkwB2q+rh7zGacm4+EgKCqzuj/2zHH0t4ZonFfK1v3th3+ld/15d921JfrwDI/I6pKOGVkFXOmlTCyupiR1SWMrCph6ICirPiCPNQRdILEgXZ2NrfzwYHDQWPnwXaWbtrLruZ2OkO9V1lEnD9SEcEjIAghVe55voHqUj8XTRrCJ6YMteBg8lavN8gRES/wLnAB0AgsA+ZF3yFKRL4MTFXVG0VkLvApVb1SREqAgKoGRWQosBIY5j7fDMxQ1d3xFnbGjBlqS1f0LBxWdja3s2XP4eacxqgmnl3NHUekLyrwMMr9gh9Z7TxGuY8RVcV50/QSDisdwTCR7gePSNeXf9d2D30T7Z0hXljfxJOrd/Dsup20BkJucBjMJ6YMs+CQIarKxt0tlBX6qC0rxOOxvqVjEZHl8fzojud//GlAg6pudDNeBMwBom8ZOAe4w91+DLhHRERVW6PSFOHUwk034bDS2hmiNRCktSNEayBEW2eQlm7bbQHneWsg6P7rbB/qCLJtXxuN+9oIhMJd+YrAsAHFjKgq5twTa50v/65HMbVlhcdFJ63HIxT7vX06tqjAy6zJQ5g1eUhXcHhq9Q6eWLGd37+5tSs4XDJlKGeOrbHgkAb7WwPc8qfVPL3mAwAKvMLgiiJGVZdwQm0ZJ9SWMnHYAD5cV3Vc/H0nUzwBYTiwNep5I3B6T2ncX/8HgBpgt4icDjwIjAauUdVIu4QCfxcRBX6uqveTxVSdX5mRL+G2QIiWbttt3b6oWwMh5wu+03mtpdt2mxsE2jvDvRcgSqHPQ4nfS4nf5/xb6OOkoeVcMGkwI6sO/8ofVlmM32dfUMnSU3BY7AaHqpICLpo0hHGDyigs8FLk81Ds91Lk81JU4KWowNP1b6HPS2VJAeVFBZl+WznljY17+NofVtDU3MFXZ45nYHkh2/e3sWN/G+/vbeXPK7bR3O58xUwfWcn/mjWBj5wwMMOlzh3xBIRYIbb7L/0e06jqUmCSiJwMPCQiT6tqO3CWqm4XkUHAP0Tkn6r60lEnF7kBuAFg8Ig6Xli/K44ixy5Me+TLujNEa0fQ/fUdoqUj2PXruyUQ7PGXeCIjZ7weoaTAS7HfS2mhj+ICLyV+LwOKCxhaUeR+kUd9qfu9FPt9lHbbLna/+CPbxQVe+xWaBboHhxffbeLJVTv4y8rttARCceXh9QiXTR/OgvNOYGxtWYpLnLtaA0Heen8/z6zbyUOvb6auppTHv3wWU0YMOCqtqtJ0qIPn1u3iv559j6t+sZSzxw/k1tkTOXFwefoLn2Pi6UM4E6cz+CL3+S0AqvofUWmWuGleFxEf8AFQq90yF5HngW+pan23/XcAh1T1R8cqS+HQ8Tr0up/G+97iUlTgodTvc794j/yCjmwX+71HpOkxfaGvKwgU+jxWXT0OhcJKSyBIe2eIjs4w7Z0h2jvDtAdDtHc6zX7tQWf/O9sPsmjZFjqCYT4xZSgLzx/HSUMqMv0Wssajy7byyJtbWLPtAMGw4hH47IyR3Dp7Ylz9W+2dIX77xvvc83wDLR1Bbjz3BBacN46igr41H+ayePsQ4gkIPpxO5ZnANpxO5atUdW1UmgXAlKhO5U+r6mdFZAyw1W1GGg28DkwF2gCPqjaLSCnwD+BOVf3bscoyceop+tu/Ptfbe+pR5Fd61xd8gdc6o0xG7T7UwQOvbOI3r22mJRDigomDuen8cUwdUZnpomXUs+t2Mv+hek4eWsF5E2o5fWwNp46uoqwPAx32tgT47pPv8Ke3tjFmYCn//qkpnHlCTQpKnbhwWAmrprzWn7SA4GZ2CfBTnGGnD6rq90TkTqBeVReLSBHwMHAKsBeYq6obReQa4GagEwjjfOn/WUTGAo+72fuAR1T1e72Vw0YZmXy1vzXAr1/bzIOvbOJge5BzT6zlpvPHMaOuOtNFS7ste1qZ/X9fZlRNCY/d+JGk/aJ/5b3dfPvPq3l/TytXnT6Kmy8+iYoM9uFs3dvK1b9cypa9rXg9Ql1NCeMGlfH+nlYOtHXyxMKzGFRelJRzJTUgZAsLCCbfNbd38ts3tvDLlzeypyXAGWOrufnik5k+8vioMbR3hrj8Z6+xdW8rT37lbEZWlyQ1/7ZAiP/zj/U88MomBpUX8cDnZjBp2NF9Eam2q7mdz973OvtaO/n8WXV0BMO8t/MQG5oOMaSiiNc37uGuOZO45sy6pJzPAoIxOawtEOL3b27hvhc30HSog2vOGM03L5qQ0V+06XDz/6xi0bKtPHDdDGaePDhl51mxdT/zf72MicMqeHh+90GTqfOjJet5fv0utu5tpTOk/Pb60zl1dNURaVSVmT9+keFVxUkrW7wBwYarGJOFiv1evvDRMTz7jXO57sw6fvvG+3z8xy/y5Kod5NKPuET8sX4ri5ZtZcF5J6Q0GIAzJHX+2WN4+b3drN1+IKXniqjfvJd7nm+g0Odh1uQhPDz/tKOCATiTJC+cNITXN+zhQGtnWsoWYQHBmCxWXlTAHZdO4s8LzmJQRSELHnmLL/x6GVv3tvZ+cI757xc2MH1kJV+/YEJaznf16aMp9Xu5/6WNaTlfZAmU315/Oj/8zLRj9g9dNGkwwbDy3PqdaSlbhAUEY3LA1BGV/PnLZ3Hr7Im8uWkvF/zkRe57cQOdocQmNWaz5vYgJw+twJumkX8DiguYd9oo/rpqB437Uhtg12w7wAvrm5j/0TGU+HsfKTVtRCWDygtZssYCgjEmBp/Xw/yPjuEfXz+Xc8bX8v2n/8kn/+8rrNi6P9NFS4pAMERhmmfWf+GjYxDggVc2pfQ89z7fQHmRj2vOHB1Xeo9HuGTKUJ775y7e29mc0rIdcd60nckYkxTDKou5/9oZ3H/NqRxs6+SzP3+dp1fvyHSx+i0QCqd9qZVhlcVcOm0Yf1i2lf2tgZSd5/n1u7hs+vCEBgUsPH8cpYVevvXYqiOWnk8lCwjG5KgLJw3hya+czeRhFXz5kbd4+PXNmS5SvwSCYfwZWJblhnPH0hoI8dBr76fsHJ0hZUBxYiPEBpYVcselk1ixdT+/fm1zagrWjQUEY3JYVamf311/BjNPGsStT6zlR0vW5+QopGDIucteJhZjPGlIBR8/eTAPvrop5g2X+kvVuZNgX1ZFuHTaMD40qpInVmxLerlisYBgTI4r9nu5719O5coZI7nn+Qb+//9ZRTDHOpsjy7ZnanXem84fx4G2Tn77RvJrCZHWHl8fAoKIcNa4gazdfpCWFASr7iwgGJMHfF4P3798Cl85fxyP1jfyxYeX0xbnqqvZIBB0A0KGVvKdNrKSc06s5Zcvb0z65xYMO++tr6OnZtRVEworb29J/eABCwjG5AkR4esXTuCuyybz3PpdXPXLN9jXkrqO0mSKBITCgsx9Jd10/jh2Hwrw+ze3JDVfNx70OSB8aFQlHoFlm/cmsVSxWUAwJs9cc8Zofnb1h1i7/SCX3/daysfYJ0NHhmsIAB+uq+b0MdX8/KUNSe1LiNQQ+tJkBM7kxJOGVFD/vgUEY0wfzJo8lIe/cBpNzR185mev824ax7L3RVdAyPAd/r510QR2Hwqw4HdvJW3SX6SG4OnH/VFOG1PN21v2p3wiogUEY/LU6WNrePSLZxJW5Yr7Xmd5Gn5h9lVXk1GGA8KMumq+e9lkXny3iVv/vCYpI7a6agjevgeEGXVVtAacmyqlkgUEY/LYyUMr+J8vfYTqUj9X/3Ipz65L71II8cr0KKNo804bxcLzxrFo2Vbue7H/6xyF3KDSnxrC6WNqKPR5+M8l61M6SS3zn74xJqVGVpfwxxvPZPygcm54eDl/rN+a6SId5fAoo+y4veU3LjyRmScN4r4XN/R7CG/kC7yvfQgAteWF3HXZZF5p2M3/+cf6fpXnWCwgGHMcGFhWyO9vOIMzx9bwrcdW8fMXN2S6SEcIZEkfQoSI8JlTR3CgrZPl7+/rV16RgNDf2/V+dsZIrjh1BP/9wgY+ONDer7x6kh2fvjEm5coKfTzwuRnMnjqU/3j6n3zvyXeyZlZzIOSM/c+WgABw9om1FHiFZ/+5q1/5JKOGEPHFc09AFf7+zgf9ziuW7Pn0jTEpV+jzcvfcU7j2zNH84uVNPLU6NV8sicr0xLRYygp9nDG2hmf62e8SCQjJWNZ73KAyxg8q4+kUXbfs+fSNMWnh8Qi3f3ISYweW8rMXG7KilpAtw067m3nSIDY2tbBpd0uf80hmQACYNXkISzftYW8KJh3G9emLyCwRWS8iDSJyc4zXC0XkD+7rS0Wkzt1/moiscB8rReRT8eZpjEkdr0f44rljWbPtIC+/tzvTxcmaYafdRW7l2Z/RWZFRRt5+jDKKdtGkIYQVnnkn+SPGev30RcQL3AtcDEwE5onIxG7J5gP7VHUc8BPgB+7+NcAMVZ0OzAJ+LiK+OPM0xqTQp04ZwZCKIv77hYZMFyWrhp1GG1ldwomDy3iuH/0IwVByawiThlUwvLKY59f3r28jlng+/dOABlXdqKoBYBEwp1uaOcBD7vZjwEwREVVtVdXIHPAiIFI3jSdPY0wK+X0erj97DG9s3MtbW/o3kqa/srEPIWLmyYN5c9NeDrb37Yb3YU1uQBARJg2rSMns83g+/eFA9MDlRndfzDRuADgA1ACIyOkishZYDdzovh5PnsaYFJt32igqSwr42QuZHYaaDYvb9WTmSYMIhpWX3m3q0/HBJPchgNO5/P6e1qQvZRHPpx/rXXTvheoxjaouVdVJwIeBW0SkKM48nYxFbhCRehGpb2rq2wUxxsRWWujjujPr+Mc7OzO63lE2LG7Xk1NGVVFe6OO1DXv6dHyyO5XBCQjBsPL+nr53dscSz6ffCIyMej4C2N5TGhHxAQOAIxZOUdV1QAswOc48I8fdr6ozVHVGbW1tHMU1xiTicx+po8Tv5b4M1hICwTAece7rkG28HuHDY6p5Y2N2BQSAhl3pDwjLgPEiMkZE/MBcYHG3NIuB69ztzwDPqaq6x/gARGQ0MAHYHGeexpg0qCr1M++0UTyxcnvGlsoOhMJZ16Ec7Yyx1WxsamHXwcRnCHcFhCSNMgIYW+sEhA1Nh5KWJ8QRENw2/4XAEmAd8KiqrhWRO0XkUjfZA0CNiDQAXwciw0g/CqwUkRXA48CXVXV3T3km840ZY+J3/dlj8Aj84qX+L+bWF4FgOCubiyLOGFsDwBubEl8xtmumcj9WO+2urNDH0AFFNOxKbkDwxZNIVZ8Cnuq277ao7XbgihjHPQw8HG+expjMGDqgmE+dMpxFy7Zy08zxDCwrTOv5O4Jh/L7sWNgulolDKygv9PHGxj1cOm1YQsd2rWWUxBoCOM1GyQ4I2RuSjTFp9cVzTyAQCvOrVzel/dyBYDjrJqVF83k9fe5HOLyWUXLf3wm1ZWxoOkQ4icthZ+8VMMak1Qm1ZVw8eQi/ef19mvs45r6vsr0PAfrejxDsWu00ueUZN6iM1kCID/rQr9GT7L4Cxpi0+tK542huD/LQa5vTet5AMJTVfQjQ936EyMS0ZNcQIiONktmxnN1XwBiTVlNGDODCiYO59/kNbN2bvhFHgWD21xCi+xEScXhiWnLLM6q6BIAtSbxO2X0FjDFpd/ulkxCB2xevTdtKqLnQZNTXfoRwV0BI7vsbXFFEgVfYurctaXlm9xUwxqTd8Mpivn7BiTz3z138bU167peQ7cNOI2bUVbGxqYX9rfEvPR1M4g1yonk9wvDKYrYmce5I9l8BY0zafe4jdUwcWsEdf1mblg7mQDCclesYdTd9RCUAKxsPxH1MOEm30IxlZHUJjdZkZIxJJZ/Xw79/egq7mjv48d/fTfn5OnKkhjBlxABEYOXW/XEfk6oaAjgBwfoQjDEpN31kJdeeMZqHXt+c0BdgX+RCpzJAeVEB42rLEvo8IjfISfbENICRVSXsa+3kUEew98RxyP4rYIzJmG9cNIHaskL+9+OrCSZ5qeVoHTkSEACmjaxkxdb9cXe4h9zPLTU1hGKApI0Iy40rYIzJiIqiAm7/5CTWbj/IQ6+/n7LzBELZPVM52rSRlexpCdC4L77RPe4N01LTh1DlDD21gGCMSYtLpgzhvAm1/Pjv69m+P3lDHKPlyigjiO5Yjq/ZKBROZQ3BDQhxBqfe5MYVMMZkjIhw55zJdIbC/DpFM5hzpQ8B4KSh5fh9nrj7EVJxx7SIqpICSv1eqyEYY9JnZHUJZ4+v5clVO5K6mFpELkxMiyjwepg8rIIVcQaEcAoDgogwsrrEAoIxJr1mTx3Ktv1tvL11X1LzDYWVUFjxe7N3+evupo2sZPW2A3F1tAdTcIOcaMkcemoBwRgTlwsmDsbv8/CXlTuSmm8gcj/lHKkhgDMkt70zzLs7e19YLhxWRFLTqQwwtraUzXtakjIKLHeugDEmo8qLCjhvQi1Prt7RtcZ/MuRiQJiWQMdyMKwpqx0AjB9UTmdIk1JLyJ0rYIzJuE9OG0ZTcwdv9uFWkj3pCIWA3AoIo2tKKC/ysWZb70tYhFRT0n8QMd5dBvu9JNw9LXeugDEm484/aRDFBV7+ump70vKM1BByZR4COJ25k4cNiC8ghFIbEE5wA0IybqeZO1fAGJNxJX4fH584mKfXfJC0mcu5GBDAWddo3QfNdPbyOaS6hlBW6GPYgKL0BQQRmSUi60WkQURujvF6oYj8wX19qYjUufsvEJHlIrLa/ff8qGNecPNc4T4G9fvdGGNSbvbUoextCfDahsTvLxxLR6QPIUcmpkVMHj6AQDDMe710LIfCqQ0I4NQS3tvV3O98er0CIuIF7gUuBiYC80RkYrdk84F9qjoO+AnwA3f/buCTqjoFuA54uNtxV6vqdPexqx/vwxiTJueeWEt5oY+/rExOs1EudioDTB5WAdBrs1EorCmZpRxt/KByNuxq6fcckXiuwGlAg6puVNUAsAiY0y3NHOAhd/sxYKaIiKq+raqRv5q1QJGIFParxMaYjCoq8HLBpMEsWfsBHcFQv/MLhHIzINTVlFJW6GN1HAEhFSudRhs/uIy2zhDb+rm0SDxXYDiwNep5o7svZhpVDQIHgJpuaS4H3lbVjqh9v3Kbi24VSfEnZoxJmk9OHcbB9iAvv7u733kFcrTJyOMRJg2rYM32zNcQxkU6lpv6148QzxWI9U6610uOmUZEJuE0I30x6vWr3aaks93HNTFPLnKDiNSLSH1TU1McxTXGpNpZ4wZSWVKQlNFGudpkBE4/wrodB4/ZwR4Ka8ompUWMHVgKwKamln7lE88VaARGRj0fAXT/K+hKIyI+YACw130+AngcuFZVN0QOUNVt7r/NwCM4TVNHUdX7VXWGqs6ora2N5z0ZY1LM7/Mwa9IQ/vHOTto7+9ds1JHDAWHK8AG0d4aP+cs8pKmvIVSX+ikq8KSlyWgZMF5ExoiIH5gLLO6WZjFOpzHAZ4DnVFVFpBJ4ErhFVV+NJBYRn4gMdLcLgNnAmn69E2NMWn1y2jBaAiGe/2f/xoNE+hBybdgpODUEgDXbDvaYJpiGGoKIMKKqhG39XAa71yvg9gksBJYA64BHVXWtiNwpIpe6yR4AakSkAfg6EBmauhAYB9zabXhpIbBERFYBK4BtwC/69U6MMWl1+phqBpb5+Us/m40O9yHkzuJ2EWMGllLi9x5zpFE4DX0IAMMri2nc37/lK3zxJFLVp4Cnuu27LWq7HbgixnHfBb7bQ7anxl9MY0y28Xk9fGLKUH6/bCv7WgJUlfr7lE8u9yF4PcJJQ8r55wfHriF4Pal/byOqilkV5017epJ7V8AYkzXmnT6KQDDMY8sb+5xHIJh7axlFG1ldcsy2e2diWurLMaKqhH2tnRzqCPY5j9y8AsaYrHDSkApmjK7id0vf7/OkqFydhxAxrLKYDw6097gCbChNNYThVcUA/epHyM0rYIzJGv9yxmg272nl1Q19m5OQq2sZRQyvLKYzpDQ1d8R8PRRWvGmYZTUiEhD60Y+Qm1fAGJM1Lp4yhOpSP7994/0+HR8IhhFJzU3o02F4ZeSLOPYvc2diWhr6ENxyNFoNwRiTKYU+L1fMGMEz63bxwYH2hI/vCIbxez3k6mIFkaaa7ccICGmIBwwsK8Tv81hAMMZk1tWnjSasyu/f3JLwsR3BcM72H4DThwDHqCFoemoIHo8wvLLY+hCMMZk1qqaEc8bXsmjZll7vD9BdIBTO2f4DcO5HMKC4oMcaQjompkWMqCqmcZ/1IRhjMuxfzhjNzoMdPLtuZ0LHBdwmo1w27Bi/zNM1MQ1g2IBitveh2S4it6+CMSZrnDehlqEDivjtG4k1GwVyvMkIYHhlUY9NRsE0LH8dUVXqZ19LANW+DQHO7atgjMkaPq+HeaeN4pWG3WzaHf+qm/kREIp7DAjprCFUlxYQDGufJ6fl9lUwxmSVuR8eic8j/C6BIaiBUO4HhGGVxTS3BznY3nnUa8FwOOW30IyoKnGWD9nXcnQ54pHbV8EYk1UGVRRx4aTB/HF5Y48zd7vLhz6EYw09DStpCwjV7npSe1sDfTo+t6+CMSbrnDdhEAfaOnl/T3zNRvnQZBQZehorIKS1huAGhH0WEIwx2WDCkHIA3t3ZHFf6jlAYvy/3lr6OFpklHGukUTicvhrC4SYjCwjGmCwwblAZIrD+g/ju7xsI5vY8BHBnCXs9bNt/9JDPYDiMN02jjKrdgLDXAoIxJhuU+H2Mqi6Ju4YQCIZyvsnI4xGG9jD0NBQGbzpWtwPKi3x4PWJNRsaY7HHi4HLWx9tkFAxTmOOdyuBOCosZENJXQ/B4hKqSAva12igjY0yWmDC4nE27W+hwb35zLPnQqQzOSKNYfQjO/RDSt3BfZYnf+hCMMdnjxCHlhMLKxqbeRxrlwzwEcEYa7WxuP2otp3QHhOoSv/UhGGOyx4TB8Y80yod5CAC15YWoHj3CJ5jGmcoAVaUFqe1DEJFZIrJeRBpE5OYYrxeKyB/c15eKSJ27/wIRWS4iq91/z4865lR3f4OI3C25uhi6MeYoYwaW4vMI6z+IMyDkQQ1hoDsHYPehI7+Mw5rmGkKpP3V9CCLiBe4FLgYmAvNEZGK3ZPOBfao6DvgJ8AN3/27gk6o6BbgOeDjqmJ8BNwDj3cesPr0DY0zW8fs8jK0t7bWGEA4rwbDmR0AoLwRg96Ejb6UZTHOTUVVJ3xe4i+cqnAY0qOpGVQ0Ai4A53dLMAR5ytx8DZoqIqOrbqrrd3b8WKHJrE0OBClV9XZ1S/wa4LOHSG2OyVjwjjQJue3s+BIQat4awp+VwQAiHFU3j0hXgBIRgWGnuwwJ38VyF4cDWqOeN7r6YaVQ1CBwAarqluRx4W1U73PSNveRpjMlhEwaXs3VvGy3H+GLqCLoBIQ/6EGrKnBrCnqgmo5D7Kz1dw04havmKPnQsx3MVYr2T7nWRY6YRkUk4zUhfTCDPyLE3iEi9iNQ3NTXFUVxjTDaILGHx3q6eZywH3ICQ6zOVASqKfPg9lJgsAAAc60lEQVS9HpqimowiC/yla2IaOEtgA33qR4jnKjQCI6OejwC295RGRHzAAGCv+3wE8DhwrapuiEo/opc8AVDV+1V1hqrOqK2tjaO4xphs0LWm0TE6lvOpyUhEqCnzH1lDCGeghtCP9YziuQrLgPEiMkZE/MBcYHG3NItxOo0BPgM8p6oqIpXAk8AtqvpqJLGq7gCaReQMd3TRtcATCZfeGJO1RlaVUFTgOWY/wuEaQm4vbhfhBISoGkKkySjNfQjQt/WMeg0Ibp/AQmAJsA54VFXXisidInKpm+wBoEZEGoCvA5GhqQuBccCtIrLCfQxyX/sS8EugAdgAPJ1w6Y0xWcvjEU4cXH7MkUaRgJAPNQRwFrmLHnYaCqU/IAwodpqMYt2spze+eBKp6lPAU9323Ra13Q5cEeO47wLf7SHPemByIoU1xuSWEweX8+K7Pff9BfKoUxmgprTwiCaySA0hnRPTyoucr/WDbakZZWSMMX0yYXA5Tc0dPTZfRNY6ypsaQrmf3VFzACJ9CJ40BgSf10Op39unGkJ+XAVjTFY6sZeb5eRdk1FpIYFguGsOQCQgpLOGAE6z0cE2CwjGmCzS25pGHXk0ygicTmU4PBehq4aQ5pV5KooLOGABwRiTTQZXFFJR5OtxTaN860MY2DU5zRlp1FVDSOM8BICKogJrMjLGZBcRYcKQnkca5dPENDhcQ4isZxTMWA3BZ53Kxpjsc+LgctZ/0BxzsbW860Moiyxw5zQZhbtGGaX3/VkNwRiTlSYMKedge5CdBzuOei2fZiqDs/Q0HO5DCHbNQ0hvOSqsU9kYk41OdDuWY81Yzrc+hAKvh8qSgq4mo66lK9JdQyguoLkjSDic2BLY+XEVjDFZKxIQYq1plG9NRuAsgx1ZAvvw0hXpLUNFkQ9VEl4CO3+ugjEmK1WX+qktL4xdQ8izJiM4cvmKUNh5f5moIQAJNxvlz1UwxmStEweX8V6MgJBP90OIcAJCpMnI2ZfO1U7B6VSGxNczyp+rYIzJWkMqio+61zAcvp9yPt1SPXoJ7GBXDSH9M5Uh8fWMLCAYY1KuqqQg5npGHcEQhXlUOwCnhnCgrZNAMEw4UkNIc0CoKHYXuLMagjEm21SV+mnrDNHeGTpif6SGkE8ik9P2tgQyVkPoajKyPgRjTLbpuotX65G1hLwMCKWRyWkdURPT0r+WEZDwekb5dSWMMVmp6z6/LUd+QQVC+RcQqkqc97q/tTNqYlp6A0J5oQ8RONhufQjGmCxTeawaQp71IZT4nfb79s5QVw0h3QHB4xHKC30JNxnFdcc0Y4zpj8iSDsdDk1FRgfN+2jpDRAZPpTsggLt8hXUqG2OyTWVJpMmoW0DIwyajogIv4NQQDi9dkYGAUFRgw06NMdmnsjhSQzjyF2tHHjYZFftjBIQMzLOoKPZZDcEYk338Pg/lhb7josmo2K0htGW4hlBelPiKp3FdCRGZJSLrRaRBRG6O8XqhiPzBfX2piNS5+2tE5HkROSQi93Q75gU3zxXuY1BCJTfG5JTK0oKjm4yC4by5OU5EpMmoLRDOaEAo9HnojKydEadeO5VFxAvcC1wANALLRGSxqr4TlWw+sE9Vx4nIXOAHwJVAO3ArMNl9dHe1qtYnVGJjTE6qLvEf1WQUCIUp9HkzVKLU8HoEv9fj1BAyNA8BnPWhOkPJX/76NKBBVTeqagBYBMzplmYO8JC7/RgwU0REVVtU9RWcwGCMOY5VlviPiyYjcEYaRfcheDIQEAq8idcQ4rkSw4GtUc8b3X0x06hqEDgA1MSR96/c5qJbpYfVrUTkBhGpF5H6pqamOLI0xmSj6tIeAkKedSqD07Hc3hnqmpiWiRpCgU9SEhBivZPu9ZB40nR3tapOAc52H9fESqSq96vqDFWdUVtb22thjTHZqbKk4KiZyh3BUF7WEIoLvLRFTUzLRA3B5/F03YAoXvFciUZgZNTzEcD2ntKIiA8YAOw9Vqaqus39txl4BKdpyhiTp6pL/BzqCB7xJZW/TUZe2gIhguEM9iH4UtOHsAwYLyJjRMQPzAUWd0uzGLjO3f4M8Jyq9lgSEfGJyEB3uwCYDaxJqOTGmJxS6c5W3h/VbJSPE9PAaTKKHnbqycA8hAJv4k1GvY4yUtWgiCwElgBe4EFVXSsidwL1qroYeAB4WEQacGoGcyPHi8hmoALwi8hlwIXA+8ASNxh4gWeAXyRUcmNMTqkuOTw5bVBFEeGw0hnSvOxDKPJ56eg8POw0I30IXg/BsBIOx19LiGstI1V9Cniq277borbbgSt6OLauh2xPja+Ixph8EFkFNNKxnI/3U44o9ntpau7I6DyEAjfQdobjryXk35UwxmSlrhVPW44MCPk2MQ0OdyqHwopHyMgtQiM1r2AC/Qj5dyWMMVnp8IqnzkijSOdyPtYQIp3KIdWM1A7A6UMAEupHyL8rYYzJSpXdm4wiASEP+xCK/YcnpmUqIPjczzVgAcEYk22KCryU+L2Hm4zyuYbg8x4OCBloLoLDgTaRoaf5dyWMMVmrqsTP3tbufQj5tZYRHDnsNGNNRj63ySiByWkWEIwxaVNVWsD+46QPIazQFghlsA8hUkOwgGCMyUJVJX72uk1GHXkcECL3RDjUEcTrycz7K7A+BGNMNqsq8XfNVO4IhoB87VQ+HBAyMSkNrA/BGJPlqkoKumoI+dxkFKkhtHQErcnIGGNiqSzxc7A9SDAU7goI+TgxrajAeU+HMhoQbB6CMSaLRSanHWjrzOulK4qO6EPI7DwEazIyxmSl6MlpeT0xLQuajLr6EGzYqTEmG0UvX5HXfQhRncqZmpjWNQ/BmoyMMdmoyl3gbm9LIK+bjCI1hM5QJtcysmGnxpgsVhV1k5x8riFE+hAgM0tfQ9+GncZ1PwRjjEmGyD0R9rZ0dt1vOB/7ELIhINiwU2NMVisu8FLo8xxRQ8jHYaeRPgTIZECwPgRjTBYTka7lKwKhMH6vJyM3j0m1oqggl+lhpwEbZWSMyVaVJQVdo4zysf8AnC/jSFNYppe/DiZwT+X8vBrGmKxVXervmoeQrwEBDs9W9nkz3GSU7BqCiMwSkfUi0iAiN8d4vVBE/uC+vlRE6tz9NSLyvIgcEpF7uh1zqoisdo+5W/Kx3miMOUpViRMQOoKhvOxQjoh0LHsy9NXm9QgiSe5DEBEvcC9wMTARmCciE7slmw/sU9VxwE+AH7j724FbgW/GyPpnwA3AePcxK+5SG2NyVuSeCPleQ4h0LGdqtVMRocDrIZDkpStOAxpUdaOqBoBFwJxuaeYAD7nbjwEzRURUtUVVX8EJDNEFHQpUqOrrqqrAb4DL4i61MSZnRZbAbu/M84AQqSFkKCCA04+Q7FFGw4GtUc8b3X0x06hqEDgA1PSSZ2MveQIgIjeISL2I1Dc1NcVRXGNMNqsq8RNW2H2o47hoMspUDQGcfoRkB4RY76Z7HSSeNH1Kr6r3q+oMVZ1RW1t7jCyNMbmgqtSZnLazud1qCClWkIIaQiMwMur5CGB7T2lExAcMAPb2kueIXvI0xuShyHpGOw925HVA6BpllOGAEAgmtw9hGTBeRMaIiB+YCyzulmYxcJ27/RngObdvICZV3QE0i8gZ7uiia4En4i61MSZnRQJCIBjOy1nKEZFO5UzNQwCnySgYjr+G0OtaRqoaFJGFwBLACzyoqmtF5E6gXlUXAw8AD4tIA07NYG7keBHZDFQAfhG5DLhQVd8BvgT8GigGnnYfxpg8F1kCG/JzHaOISB9CpmYqQ+JNRnEtbqeqTwFPddt3W9R2O3BFD8fW9bC/Hpgcb0GNMfkhcpMcyM+VTiOKsyQgJLvJyBhjkqas0NfVrp7XTUbZEBB8ye9UNsaYpBGRrvsi5HUNIcMT0wD8KRh2aowxSRW5L0I+B4SiPB12aowxSRUZaeT3entJmbuyYWKaLwVLVxhjTFJ1BYQ8riFkw8Q0v1eSv9qpMcYk0/HRh5AdE9OsycgYk9UifQjHxSijjE5M89gNcowx2S0yOS2fJ6YVdg07zdx7dOYhWA3BGJPFKo+jPoRMxjy/z4adGmOy3PEw7LQ4S2oIFhCMMVmt6jhoMupa3C6Db9EJCNaHYIzJYmNqShlcUci4QWWZLkrKZEMNwecVAsle3M4YY5KpqtTP0v/98UwXI6VK3BqC35vZW2gm0qlsAcEYY1KgpqyQ/5o7nXPGZ+5OjwUJtldZQDDGmBSZMz3mreLTJtGAYH0IxhiTpwoSbK6ygGCMMXkq0WG9FhCMMSZPWZORMcYYIPGF9SwgGGNMnkpJk5GIzBKR9SLSICI3x3i9UET+4L6+VETqol67xd2/XkQuitq/WURWi8gKEalPqNTGGGN6lfRhpyLiBe4FLgAagWUislhV34lKNh/Yp6rjRGQu8APgShGZCMwFJgHDgGdE5ERVDbnHnaequxMqsTHGmLikog/hNKBBVTeqagBYBMzplmYO8JC7/RgwU0TE3b9IVTtUdRPQ4OZnjDEmxVIx7HQ4sDXqeaO7L2YaVQ0CB4CaXo5V4O8islxEbujp5CJyg4jUi0h9U1NTHMU1xhgDiS8eGE/qWCGm+/J5PaU51rFnqeqHgIuBBSJyTqyTq+r9qjpDVWfU1mZuCrgxxuSauoGl/NsnTo47fTwBoREYGfV8BLC9pzQi4gMGAHuPdayqRv7dBTyONSUZY0xSDass5vqzx8adPp6AsAwYLyJjRMSP00m8uFuaxcB17vZngOdUVd39c91RSGOA8cCbIlIqIuUAIlIKXAisibvUxhhjkq7XUUaqGhSRhcASwAs8qKprReROoF5VFwMPAA+LSANOzWCue+xaEXkUeAcIAgtUNSQig4HHnX5nfMAjqvq3FLw/Y4wxcRLnh3xumDFjhtbX25QFY4xJhIgsV9UZvaXL+eWvOzs7aWxspL29PdNFMVGKiooYMWIEBQUFmS6KMSZOOR8QGhsbKS8vp66uDrcJymSYqrJnzx4aGxsZM2ZMpotjjIlTzq9l1N7eTk1NjQWDLCIi1NTUWK3NmByT8wEBsGCQheyaGJN78iIgGGOM6T8LCEng9XqZPn06kydP5oorrqC1tTUp+e7cuZPZs2czbdo0Jk6cyCWXXALAmDFjWL9+/RFp//Vf/5Uf/vCHvPDCCwwYMIBTTjmFCRMmcM455/DXv/6113O9/fbbXH/99QmX8Zvf/CbPPfdcwscZY7KPBYQkKC4uZsWKFaxZswa/3899990X97HBYLDH12677TYuuOACVq5cyTvvvMP3v/99AObOncuiRYu60oXDYR577DGuvPJKAM4++2zefvtt1q9fz913383ChQt59tlnj1mOf//3f+emm26Ku9wRN910U1e5jDG5LedHGUX7zl/W8s72g0nNc+KwCm7/5KS405999tmsWrWKzZs3M3v2bNascSZg/+hHP+LQoUPccccdfOxjH+MjH/kIr776KpdeeinXXnstN954I1u2bAHgpz/9KWeddRY7duzgwgsv7Mp76tSpAMybN48rr7yS22+/HYCXXnqJuro6Ro8ezaZNm44oz/Tp07ntttu45557mDlzJn/84x/5zne+g9frZcCAAbz00ks0NzezatUqpk2bBsChQ4e46aabqK+vR0S4/fbbueyyy5g/f37Xvi984Qt87WtfY/To0ezZs4cPPviAIUOG9P2DNsZkXF4FhEwLBoM8/fTTzJo1q9e0+/fv58UXXwTgqquu4mtf+xof/ehH2bJlCxdddBHr1q1jwYIFXHnlldxzzz18/OMf5/Of/zzDhg1j6tSpeDweVq5cybRp01i0aBHz5s3r8Vwf+tCH+M///E8A7rzzTpYsWcLw4cPZv38/APX19UyePLkr/V133cWAAQNYvXo1APv27WPFihVs27atK8BFjo3k/+qrr3L55Zcn+IkZY7JJXgWERH7JJ1NbWxvTp08HnBrC/Pnz2b69+/p/R4o07wA888wzvPPO4fsNHTx4kObmZi666CI2btzI3/72N55++mlOOeUU1qxZQ21tLfPmzWPRokVMmjSJJ554gjvvvLPHc0XPRj/rrLP43Oc+x2c/+1k+/elPA7Bjxw6iV5J95plnjmiSqqqqYuzYsWzcuJGbbrqJT3ziE0fUXAYNGtTr+zXGZL+8CgiZEulDiObz+QiHw13Pu4/JLy0t7doOh8O8/vrrFBcXH5V3dXU1V111FVdddRWzZ8/mpZde4vLLL2fevHlceOGFnHvuuUydOpVBgwb1WL63336bk092lsC97777WLp0KU8++STTp09nxYoVFBcXH1E+VT1q2GhVVRUrV65kyZIl3HvvvTz66KM8+OCDXe8tVtmNMbnFOpVTZPDgwezatYs9e/bQ0dFxzJE+F154Iffcc0/X80hwee6557pGLDU3N7NhwwZGjRoFwAknnEBNTQ0333zzMZuLVq1axV133cWCBQsA2LBhA6effjp33nknAwcOZOvWrZx88sk0NDT0WJ59+/axe/duwuEwl19+OXfddRdvvfVW1+vvvvvuEU1OxpjcZDWEFCkoKOC2227j9NNPZ8yYMZx00kk9pr377rtZsGABU6dOJRgMcs4553DfffexfPlyFi5c2FXbuP766/nwhz/cddy8efO45ZZb+NSnPnVEfi+//DKnnHIKra2tDBo0iLvvvpuZM2cC8K1vfYv33nsPVWXmzJlMmzYNEeHAgQM0NzdTXl7Ov/3bv7FgwQImT56M1+vl9ttv54QTTuDzn/98V63nP/7jPwBnLamGhgZmzOh13SxjTJbL+dVO161b19UcYvruJz/5CeXl5QnPRXj88cd56623uOuuu456za6NMdkh3tVOrcnIAPClL32JwsLChI8LBoN84xvfSEGJjDHpZk1GBnCWq77mmmsSPu6KK65IQWmMMZmQFzWEXGr2Ol7YNTEm9+R8QCgqKmLPnj32BZRFIvdDKCoqynRRjDEJyPkmoxEjRtDY2EhTU1Omi2KiRO6YZozJHTkfEAoKCuyuXMYYkwRxNRmJyCwRWS8iDSJyc4zXC0XkD+7rS0WkLuq1W9z960XkonjzNMYYk169BgQR8QL3AhcDE4F5IjKxW7L5wD5VHQf8BPiBe+xEYC4wCZgF/LeIeOPM0xhjTBrFU0M4DWhQ1Y2qGgAWAXO6pZkDPORuPwbMFGcxnDnAIlXtUNVNQIObXzx5GmOMSaN4+hCGA1ujnjcCp/eURlWDInIAqHH3v9Ht2OHudm95AiAiNwA3uE8Picj6WOmAAcCBY74TGAjs7iVNrorn/efi+ZOVb3/ySfTYeNMnK539Xefm+dP5tz06noziCQix7pbefYxnT2l62h+rZhJz3Kiq3g/cf6wCAojI/ap6Qy9p6uOZvp2L4nn/uXj+ZOXbn3wSPTbe9MlKZ3/XuXn+bPjb7i6eJqNGYGTU8xFA98Xvu9KIiA8nYu09xrHx5Jmov/Tz+FyX6fefqvMnK9/+5JPosfGmT3a6fJTp957K82fD3/YRel3czv2CfxeYCWwDlgFXqeraqDQLgCmqeqOIzAU+raqfFZFJwCM4fQbDgGeB8Tg1h2PmmQr5/EvKHL/s79okS69NRm6fwEJgCeAFHlTVtSJyJ1CvqouBB4CHRaQBp2Yw1z12rYg8CrwDBIEFqhoCiJVn8t/eUXptejImB9nftUmKnFr+2hhjTOrk/FpGxhhjksMCgjHGGCALAoKIFIvIi+7sZUTkOhF5z31c18Mx/yki/xSRVSLyuIhU9nKOkSLyvIisE5G1IvLVqNfuEJFtIrLCfVwS9VpCy26IyCIRGd+fz8Pkp1z7OxcRv4i85A4qMccLVc3oA1gAfNXdrgY2uv9WudtVMY65EPC52z8AftDLOYYCH3K3y3FGOE10n98BfDPGMROBlUAhMAbYgNMB7nW3xwJ+N00kr3OBX2T6M7VH9j1y7e/cfe124OpMf3b2SN8j4zUE4GrgCXf7IuAfqrpXVfcB/8BZA+kIqvp3VQ26T9/AmcfQI1XdoapvudvNwDoOz5juSV+W3XgZ+Lj9qjIx5NrfOcCf3XKb40RGA4KI+IGxqrrZ3RVrmYze/qC/ADydwDnrgFOApVG7F7rV8gdFpKqXsvRYRlUN4/yHmhZveUz+y9G/c4A1wIfjPafJfZmuIQwE9kc9j2eZjMOJRb6NM7/hd/GcTETKgP8B/lVVD7q7fwacAEwHdgA/7qUsvZVxF84kPGMicvHvHHXmDAVEpDye85rcl+mA0AZE32cx7iUt3I642ThtnL1OphCRApz/JL9T1T9F9qvqTlUNub/uf8Hh6nJfl90oct+XMRG5+HceUQi093Zekycy3YmBU10tcrergU04HW1V7nZ1jGNm4cx+ru22fzjwbIz0AvwG+GmM14ZGbX8Npz0VnHs4RHe2bcTpUPa522M43Kk8KSqP1dF52sMeqrn3d+6+VgOsy/RnZ4/0PTJdQwD4O/BRAFXdC9yFs7bRMuBOdx8i8ksRiazXcg/OKIp/uEPo7nP3D8WpWnd3FnANcH6MYXc/FJHVIrIKOA/nPwvqLKURWXbjb7jLbqjTyRdZdmMd8KibFhEZDLSp6o6kfDImn+TU37l7zHnAU8l5+yYXZHzpChE5Bfi6ql6ThLwWAlvUWV8p7UTka8BBVX0gE+c32SsX/85F5E/ALara0z1ITJ7J+PBIVX3bnUzjjfpl0te87klWufpoP/BwhstgslCu/Z27I6P+bMHg+JLxGoIxxpjskA19CMYYY7KABQRjjDGABQRjjDEuCwjGGGMACwgmD7lLPX8zxed4ITJfQESe6m1p6nQRkV+LyGcyXQ6TmzI+7NSYXKeql/SeKjHJGJ5qTKKshmByhohc667WuVJEHhaR0SLyrLvvWREZFWc+XxGRd9zjFrn7ykTkV5HZvCJyubv/ZyJS795w5js95LdZRAaKSJ17c5pfuOn/LiLFbpoPu/m+7t74Zk2MfD7mzlV4BGcJFETkzyKy3M3vhqi0h0Tke+5n8YY7S757fne5NQb7f27iYn8oJieIyCTg28D5qjoN+CrO0g6/UdWpOCuB3h1ndjcDp7jH3ejuuxU4oKpT3P3Pufu/raozgKnAuSIytZe8xwP3quoknImKl7v7fwXcqKpnAsf65X+ae86J7vMvqOqpwAzgKyJS4+4vBd5wP4uXgP8vOhMR+SEwCPi8OgvaGdMrCwgmV5wPPKaqu6FrPaAzgUfc1x/GXSsoDquA34nIv3B4TaCPA/dGEqhz4xqAz4rIW8DbOAvBTeTYNqnqCnd7OVDn9i+Uq+pr7v5HYh8KwJvq3Kgm4isishLnBjkjcQIOQAD4a/R5oo65FahU1S+qzTw1CbCAYHKFcIx7Brji/fL7BM6X/6nAcvcOd0flLyJjgG8CM91aw5McuYx1LB1R2yGcfrpY9xzoSUvU+T+GE6jOdGsCb0edvzPqyz5ynohlwKkiUp3AeY2xgGByxrM4v9ZrANwvu9eAue7rVwOv9JaJ254+UlWfB/4XUAmU4axGujAqXRVQgfMFfcBto7+4LwV3axvNInKGuytSZkRkuIg828OhA4B9qtoqIicBZ/SQrru/Ad8HnrSb25hE2CgjkxNUda2IfA94UURCOL+WvwI8KCLfApqAz8eRlRf4rYgMwPnl/hNV3S8i3wXudTt7Q8B3VPVPIvI2sBbnPgGv9uMtzAd+ISItwAvAAXd/T0tZg/PFfqO7ZPV6nGajuKjqH91gsFhELlFVu2mT6ZUtbmdMGohImaoecrdvxrlhzVczvWS7MdEsIBiTBiJyJXALTq38feBzqtqU2VIZcyQLCCZvici9OHcRi/ZfqvqrTJTHmGxnAcEYYwxgo4yMMca4LCAYY4wBLCAYY4xxWUAwxhgDwP8DKucRUkuDNXkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "report_results('rank', scores_cold);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'YaMus': {'PureSVDs(cs)': {'col_scaling': 0.2, 'rank': 1250}}}"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "config_cold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### saving data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(experiment_name+'_coldstart', config=config_cold, tuning=scores_cold)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## cross validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:0; max-width:15ex; vertical-align:middle; text-align:right\"></span>\n",
       "<progress style=\"width:60ex\" max=\"1\" value=\"1\" class=\"Progress-main\"/></progress>\n",
       "<span class=\"Progress-label\"><strong>100%</strong></span>\n",
       "<span class=\"Iteration-label\">1/1</span>\n",
       "<span class=\"Time-label\">[02:22:47<02:22:47, 8567.25s/it]</span></div>"
      ],
      "text/plain": [
       "\u001b[A\u001b[A\u001b[2K\r",
       " [████████████████████████████████████████████████████████████] 1/1 [02:22:47<02:22:47, 8567.25s/it]\u001b[B"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div><span class=\"Text-label\" style=\"display:inline-block; overflow:hidden; white-space:nowrap; text-overflow:ellipsis; min-width:15ex; max-width:15ex; vertical-align:middle; text-align:right\">YaMus folds</span>\n",
       "<progress style=\"width:45ex\" max=\"5\" value=\"5\" class=\"Progress-main\"/></progress>\n",
       "<span class=\"Progress-label\"><strong>100%</strong></span>\n",
       "<span class=\"Iteration-label\">5/5</span>\n",
       "<span class=\"Time-label\">[02:22:47<28:34, 1713.45s/it]</span></div>"
      ],
      "text/plain": [
       "\u001b[A\u001b[2K\r",
       "    YaMus folds [█████████████████████████████████████████████] 5/5 [02:22:47<28:34, 1713.45s/it]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "result_cold = {}\n",
    "for label in track(data_labels):\n",
    "    models_cold = prepare_cold_start_recommender_models(label, data_models_cold, config_cold)\n",
    "    result_cold[label] = ee.run_cv_experiment(models_cold,\n",
    "                                              fold_experiment=ee.topk_test,\n",
    "                                              topk_list=topk_values,\n",
    "                                              ignore_feedback=True,\n",
    "                                              iterator=lambda x: track(x, label=f'{label} folds'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEWCAYAAABmE+CbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAH+ZJREFUeJzt3X90VeWd7/H3x4RftYAWsbcFNVS5VYQQbARnqdiRgtg6ooMsQW9Flw62BW+d0c7FTkWLjtXWjh0X3nFsdUppLVgst6liaZVaWpeDhPJDEGkDRY1Q5ZcUq6iB7/3jPODxcEJ2QsgJ8HmtdVb2fvaznzx7Q/LJ/vVsRQRmZmZHlLoDZmbWPjgQzMwMcCCYmVniQDAzM8CBYGZmiQPBzMwAB4KZmSUOBDtsSfqRpIcKys6RtFnSx/axXrmkkLReUlleece0bsOB7LfZgeJAsMPZ/wY+K2k4gKTOwHeBGyJiQ4b1twMj8uYvADa1ei/N2ogDwQ5bEbEZuA54QNKRwC3Amoj4vqS/kfTfkt6QtEHSvZI6FDQxA7gib/4K4Af5FSTVS/p03vztkr6fpj8k6eF0VPGGpOckHdP6W2qWjQPBDmsR8RNgMfBjYAJwbVrUAHwZOAY4ExiZt2y3nwLnSuomqQdwBvBYM779VcCHgN5AD+BLwI6WbYnZ/isvdQfM2oGJwBrgXyLiZYCIWJS3fK2kB4BzgGl55W8BTwBjgC7AHOCdZnzf98gFzkkR8TxQ2+ItMGsFDgQ77EXEa5I2ASt3l0k6Gfg28Clyf8WXAwuLrP4DcqeaupA7omiO7wMfBx6R1I3cKaivRYQvSltJ+JSRWXH/Cawg99d7N2AKoCL1fg2cABwVEc8WWf5XcoGy2//YPRER70bErRFxCnAWcDFweSv136zZHAhmxXUFtgF/lXQKe18/ACBy48dfAFzUSDtLgbHpVtXBwN/vXiDpXEn9JR0B/IXcKaSdrbgNZs3iQDAr7gZgPLlbS/8TmNVYxYhYEREvNLL4X4CTgTeAm4GH85Z9nNyF6b+QO131JLmL22YlIb8gx8zMwEcIZmaWOBDMzAxwIJiZWeJAMDMz4CB7MO2YY46JioqKUnfDzOygsnjx4k0R0bOpegdVIFRUVFBb66f7zcyaQ9JLWer5lJGZmQEOBDMzSxwIZmYGHGTXEIp57733qK+vZ8cODyPfnnTu3JnevXvToUPhO2XMrL066AOhvr6erl27UlFRgVRsMEpraxHB5s2bqa+vp0+fPqXujplldNCfMtqxYwc9evRwGLQjkujRo4eP2swOMgd9IAAOg3bI/yZmB59DIhDMzGz/ORDauYqKCjZt2rTfdczMmnLQX1Q2MzsQKiY/XuouALDuzs+12ffyEcIBsG7dOk4++WSuueYa+vfvz+WXX86TTz7JmWeeSd++fXnuuefYsmULF110EZWVlZxxxhksX74cgM2bNzNixAgGDRrEtddeS/4LjH74wx8yePBgqqqquPbaa9m5029bNLPW40A4QOrq6vjyl7/M8uXLefHFF3n44Yf53e9+x913380dd9zBLbfcwqBBg1i+fDl33HEHV1xxBQBf//rXOeuss1iyZAkXXnghL7/8MgCrVq1i1qxZPPPMMyxdupSysjJ+9KMflXITzewQ41NGB0ifPn0YMGAAAKeeeirDhg1DEgMGDGDdunW89NJLPProowCce+65bN68mW3btrFgwQJ++tOfAvC5z32Oo48+GoCnnnqKxYsXc/rppwPw9ttvc+yxx5Zgy8zsUOVAOEA6deq0Z/qII47YM3/EEUfQ0NBAefneu373rZrFbtmMCMaPH883vvGNA9RjMzvc+ZRRiQwdOnTPKZ+nn36aY445hm7dun2g/IknnmDr1q0ADBs2jNmzZ/P6668DsGXLFl56KdOItmZmmWQKBEkjJa2WVCdpcpHlnSTNSssXSqooWH68pDcl3Zi1zUPdrbfeSm1tLZWVlUyePJnp06cDcMstt7BgwQJOO+00fvnLX3L88ccD0K9fP26//XZGjBhBZWUlw4cPZ8OGDaXcBDM7xCj/LpaiFaQy4A/AcKAeWASMi4gX8up8CaiMiC9IGgtcHBGX5i1/FNgFLIyIu7O0WUx1dXUUviBn1apVnHLKKVm319qQ/23sYHYo3XYqaXFEVDdVL8sRwmCgLiLWRsS7wExgVEGdUcD0ND0bGKZ0IlzSRcBaYGUz2zQzszaU5aJyL+CVvPl6YEhjdSKiQdI2oIekt4H/Q+5I4MZi9ffRJgCSJgATgD2nT8zswDiU/iq25styhFBslLLC80yN1fk6cE9EvNmCNnOFEQ9ERHVEVPfs2eQ7os3MrIWyHCHUA8flzfcG1jdSp15SOdAd2ELur/5LJH0TOArYJWkHsDhDm2Zm1oayBMIioK+kPsCrwFjgsoI6NcB44FngEmB+5K5Wn727gqRbgTcjYloKjabaNDOzNtRkIKRrApOAeUAZ8FBErJQ0FaiNiBrgQWCGpDpyRwZjW9Lmfm6LmZnth0xPKkfEXGBuQdmUvOkdwJgm2ri1qTZbQ2tfFPPFLTM7XPhJ5VZQVlZGVVUV/fv3Z8yYMbz11lut0u5rr73GBRdcwMCBA+nXrx+f/exngdw4SatXr/5A3euvv55vfvObPP3003Tv3p1BgwbxyU9+kqFDh/LYY481+b2WLFnCNddc0+w+3njjjcyfP7/Z65lZ++NAaAVdunRh6dKlrFixgo4dO3L//fdnXrehoaHRZVOmTGH48OEsW7aMF154gTvvvBOAsWPHMnPmzD31du3axezZs7n00tyzgGeffTZLlixh9erV3HvvvUyaNImnnnpqn/244447uO666zL3e7frrrtuT7/M7ODmQGhlZ599NnV1daxbt47+/fvvKb/77ru59dZbAfj0pz/NV7/6Vc455xz+/d//nY0bNzJ69GhOP/10Tj/9dJ555hkANmzYQO/evfe0UVlZCcC4ceM+EAgLFiygoqKCE044Ya/+VFVVMWXKFKZNmwbAT37yE/r378/AgQMZOnQoANu3b2f58uUMHDgQgDfffJOrrrqKAQMGUFlZyaOPPsrOnTu58sor6d+/PwMGDOCee+4B4IQTTmDz5s38+c9/bq1daGYl4tFOW1FDQwNPPPEEI0eObLLuG2+8wW9+8xsALrvsMv7xH/+Rs846i5dffpnzzjuPVatWMXHiRC699FKmTZvGZz7zGa666io+/vGPU1lZyRFHHMGyZcsYOHAgM2fOZNy4cY1+r9NOO41vfetbAEydOpV58+bRq1cv3njjDQBqa2s/EF633XYb3bt35/nnnwdg69atLF26lFdffZUVK1bs6X9++8888wyjR49u5h4zs/bERwit4O2336aqqorq6mqOP/54rr766ibX2X16B+DJJ59k0qRJVFVVceGFF/KXv/yF7du3c95557F27Vr+4R/+gRdffJFBgwaxceNG4P2jhIaGBn72s58xZkzj1/Tzx6s688wzufLKK/nud7+7541rGzZsIP+hvyeffJKJEyfumT/66KP5xCc+wdq1a7nuuuv4xS9+Qbdu3fYsP/bYY1m/3o+RmB3sfITQCnZfQ8hXXl7Orl279szv2LHjA8uPPPLIPdO7du3i2WefpUuXLnu1/ZGPfITLLruMyy67jAsuuIAFCxYwevRoxo0bx4gRIzjnnHOorKzc58tylixZsmeQufvvv5+FCxfy+OOPU1VVxdKlS+nSpcsH+hcRe72T4eijj2bZsmXMmzeP++67j0ceeYSHHnpoz7YV67uZHVwOuUBoL7eJfvSjH+X1119n8+bNfPjDH+axxx5r9FTSiBEjmDZtGl/5ylcAWLp0KVVVVcyfP58zzjiDD33oQ2zfvp01a9bsGc/pxBNPpEePHkyePJnrr7++0X4sX76c2267je9973sArFmzhiFDhjBkyBB+/vOf88orr3DKKafw7W9/e6/+fOc73wFyp4x27txJx44dGT16NCeeeCJXXnnlnvp/+MMf9nmEYmYHh0MuENqLDh06MGXKFIYMGUKfPn04+eSTG6177733MnHiRCorK2loaGDo0KHcf//9LF68mEmTJu052rjmmmv2vEITcqeNbrrpJi6++OIPtPfb3/6WQYMG8dZbb3Hsscdy7733MmzYMAC+8pWv8Mc//pGIYNiwYQwcOBBJbNu2je3bt9O1a1e+9rWvMXHiRPr3709ZWRm33HILJ554IlddddWeo57db2577733qKuro7q6yZF1zayda/J9CO2J34dw4Nxzzz107dq12c8izJkzh9///vfcdtttey3zv83Bx6Odvu9Q2het+T4EOwx88Ytf/MB7oLNqaGjghhtuOAA9MrO2dkicMip2EdSap3Pnznz+859v9nqNXTs4mI48zSznoD9C6Ny5M5s3b/YvoHYkIti8eTOdO3cudVfMrBkO+iOE3r17U19fv+f+fGsfOnfu/IGnrM2s/TvoA6FDhw706dOn1N0wMzvoHfSnjMzMrHU4EMzMDMgYCJJGSlotqU7S5CLLO0malZYvlFSRygdLWpo+yyRdnLfOOknPp2W1hW2amVnbavIagqQy4D5gOFAPLJJUExEv5FW7GtgaESdJGgvcBVwKrACq0yszPwYsk/TziNj9EoC/jYhNrblBZs11KD2AZLY/shwhDAbqImJtRLwLzARGFdQZBUxP07OBYZIUEW/l/fLvDPjeUDOzdipLIPQCXsmbr09lReukANgG9ACQNETSSuB54At5ARHALyUtljShsW8uaYKkWkm1vrXUzOzAyRIIxR4BLvxLv9E6EbEwIk4FTgdukrT7aaUzI+I04HxgoqShxb55RDwQEdURUZ0/Zr+ZmbWuLIFQDxyXN98bKHwbyp46ksqB7sCW/AoRsQr4K9A/za9PX18H5pA7NWVmZiWSJRAWAX0l9ZHUERgL1BTUqQHGp+lLgPkREWmdcgBJJwCfBNZJOlJS11R+JDCC3AVoMzMrkSbvMkp3CE0C5gFlwEMRsVLSVKA2ImqAB4EZkurIHRmMTaufBUyW9B6wC/hSRGyS9AlgThqQrhx4OCJ+0dobZ2Zm2WUauiIi5gJzC8qm5E3vAPYa9jIiZgAzipSvBQY2t7NmZnbg+EllMzMDHAhmZpY4EMzMDHAgmJlZ4kAwMzPAgWBmZokDwczMAAeCmZklDgQzMwMcCGZmljgQzMwMcCCYmVniQDAzM8CBYGZmiQPBzMyAjIEgaaSk1ZLqJE0usryTpFlp+UJJFal8sKSl6bNM0sVZ2zQzs7bVZCBIKgPuA84H+gHjJPUrqHY1sDUiTgLuAe5K5SuA6oioAkYC/ympPGObZmbWhrIcIQwG6iJibUS8C8wERhXUGQVMT9OzgWGSFBFvRURDKu8MRDPaNDOzNpQlEHoBr+TN16eyonVSAGwDegBIGiJpJfA88IW0PEubZmbWhrIEgoqURdY6EbEwIk4FTgduktQ5Y5u5hqUJkmol1W7cuDFDd83MrCWyBEI9cFzefG9gfWN1JJUD3YEt+RUiYhXwV6B/xjZ3r/dARFRHRHXPnj0zdNfMzFoiSyAsAvpK6iOpIzAWqCmoUwOMT9OXAPMjItI65QCSTgA+CazL2KaZmbWh8qYqRESDpEnAPKAMeCgiVkqaCtRGRA3wIDBDUh25I4OxafWzgMmS3gN2AV+KiE0Axdps5W0zM7NmaDIQACJiLjC3oGxK3vQOYEyR9WYAM7K2aWZmpeMnlc3MDHAgmJlZ4kAwMzPAgWBmZokDwczMAAeCmZklDgQzMwMcCGZmljgQzMwMcCCYmVniQDAzM8CBYGZmiQPBzMwAB4KZmSUOBDMzAxwIZmaWZAoESSMlrZZUJ2lykeWdJM1KyxdKqkjlwyUtlvR8+npu3jpPpzaXps+xrbVRZmbWfE2+MU1SGXAfMByoBxZJqomIF/KqXQ1sjYiTJI0F7gIuBTYBfxcR6yX1J/fKzF55610eEbWttC1mZrYfshwhDAbqImJtRLwLzARGFdQZBUxP07OBYZIUEUsiYn0qXwl0ltSpNTpuZmatK0sg9AJeyZuv54N/5X+gTkQ0ANuAHgV1RgNLIuKdvLL/SqeLbpakYt9c0gRJtZJqN27cmKG7ZmbWElkCodgv6mhOHUmnkjuNdG3e8ssjYgBwdvp8vtg3j4gHIqI6Iqp79uyZobtmZtYSWQKhHjgub743sL6xOpLKge7AljTfG5gDXBERa3avEBGvpq/bgYfJnZoyM7MSyRIIi4C+kvpI6giMBWoK6tQA49P0JcD8iAhJRwGPAzdFxDO7K0sql3RMmu4AXACs2L9NMTOz/dFkIKRrApPI3SG0CngkIlZKmirpwlTtQaCHpDrgn4Ddt6ZOAk4Cbi64vbQTME/ScmAp8Crw3dbcMDMza54mbzsFiIi5wNyCsil50zuAMUXWux24vZFmP5W9m2ZmdqD5SWUzMwMcCGZmljgQzMwMyHgNwQ49FZMfL3UXAFh35+dK3QUzS3yEYGZmgAPBzMwSB4KZmQEOBDMzSxwIZmYGOBDMzCxxIJiZGeBAMDOzxIFgZmaAA8HMzBIHgpmZAQ4EMzNLMgWCpJGSVkuqkzS5yPJOkmal5QslVaTy4ZIWS3o+fT03b51PpfI6SfdKUmttlJmZNV+TgSCpDLgPOB/oB4yT1K+g2tXA1og4CbgHuCuVbwL+LiIGkHvn8oy8df4DmAD0TZ+R+7EdZma2n7IcIQwG6iJibUS8C8wERhXUGQVMT9OzgWGSFBFLImJ9Kl8JdE5HEx8DukXEsxERwA+Ai/Z7a8zMrMWyBEIv4JW8+fpUVrRORDQA24AeBXVGA0si4p1Uv76JNgGQNEFSraTajRs3ZuiumZm1RJZAKHZuP5pTR9Kp5E4jXduMNnOFEQ9ERHVEVPfs2TNDd83MrCWyBEI9cFzefG9gfWN1JJUD3YEtab43MAe4IiLW5NXv3USbZmbWhrIEwiKgr6Q+kjoCY4Gagjo15C4aA1wCzI+IkHQU8DhwU0Q8s7tyRGwAtks6I91ddAXws/3cFjMz2w9NBkK6JjAJmAesAh6JiJWSpkq6MFV7EOghqQ74J2D3ramTgJOAmyUtTZ9j07IvAt8D6oA1wBOttVFmZtZ85VkqRcRcYG5B2ZS86R3AmCLr3Q7c3kibtUD/5nTWzMwOHD+pbGZmgAPBzMwSB4KZmQEOBDMzSxwIZmYGOBDMzCxxIJiZGeBAMDOzxIFgZmaAA8HMzBIHgpmZAQ4EMzNLHAhmZgY4EMzMLHEgmJkZ4EAwM7MkUyBIGilptaQ6SZOLLO8kaVZavlBSRSrvIenXkt6UNK1gnadTm4VvUjMzsxJo8o1pksqA+4DhQD2wSFJNRLyQV+1qYGtEnCRpLHAXcCmwA7iZ3JvRir0d7fL05jQzMyuxLEcIg4G6iFgbEe8CM4FRBXVGAdPT9GxgmCRFxF8j4nfkgsHMzNqxLIHQC3glb74+lRWtExENwDagR4a2/yudLrpZkopVkDRBUq2k2o0bN2Zo0szMWiJLIBT7RR0tqFPo8ogYAJydPp8vVikiHoiI6oio7tmzZ5OdNTOzlskSCPXAcXnzvYH1jdWRVA50B7bsq9GIeDV93Q48TO7UlJmZlUiWQFgE9JXUR1JHYCxQU1CnBhifpi8B5kdEo0cIksolHZOmOwAXACua23kzM2s9Td5lFBENkiYB84Ay4KGIWClpKlAbETXAg8AMSXXkjgzG7l5f0jqgG9BR0kXACOAlYF4KgzLgSeC7rbplZmbWLE0GAkBEzAXmFpRNyZveAYxpZN2KRpr9VLYumplZW/CTymZmBjgQzMwscSCYmRngQDAzs8SBYGZmgAPBzMwSB4KZmQEZn0M4VFRMfrzUXQBg3Z2fK3UXzMz24iMEMzMDHAhmZpY4EMzMDHAgmJlZ4kAwMzPAgWBmZokDwczMgIyBIGmkpNWS6iRNLrK8k6RZaflCSRWpvIekX0t6U9K0gnU+Jen5tM69koq9l9nMzNpIk4EgqQy4Dzgf6AeMk9SvoNrVwNaIOAm4B7grle8AbgZuLNL0fwATgL7pM7IlG2BmZq0jyxHCYKAuItZGxLvATGBUQZ1RwPQ0PRsYJkkR8deI+B25YNhD0seAbhHxbHr38g+Ai/ZnQ8zMbP9kCYRewCt58/WprGidiGgAtgE9mmizvok2AZA0QVKtpNqNGzdm6K6ZmbVElkAodm4/WlCnRfUj4oGIqI6I6p49e+6jSTMz2x9ZAqEeOC5vvjewvrE6ksqB7sCWJtrs3USbZmbWhrIEwiKgr6Q+kjoCY4Gagjo1wPg0fQkwP10bKCoiNgDbJZ2R7i66AvhZs3tvZmatpsnhryOiQdIkYB5QBjwUESslTQVqI6IGeBCYIamO3JHB2N3rS1oHdAM6SroIGBERLwBfBL4PdAGeSB8zMyuRTO9DiIi5wNyCsil50zuAMY2sW9FIeS3QP2tHzczswPKTymZmBjgQzMwscSCYmRngQDAzs8SBYGZmgAPBzMwSB4KZmQEOBDMzSxwIZmYGOBDMzCxxIJiZGeBAMDOzxIFgZmaAA8HMzBIHgpmZAQ4EMzNLMgWCpJGSVkuqkzS5yPJOkmal5QslVeQtuymVr5Z0Xl75OknPS1oqqbY1NsbMzFquyTemSSoD7gOGA/XAIkk16TWYu10NbI2IkySNBe4CLpXUj9zrNE8FPg48Kel/RsTOtN7fRsSmVtweMzNroSxHCIOBuohYGxHvAjOBUQV1RgHT0/RsYJgkpfKZEfFORPwJqEvtmZlZO5MlEHoBr+TN16eyonUiogHYBvRoYt0AfilpsaQJjX1zSRMk1Uqq3bhxY4bumplZS2QJBBUpi4x19rXumRFxGnA+MFHS0GLfPCIeiIjqiKju2bNnhu6amVlLZAmEeuC4vPnewPrG6kgqB7oDW/a1bkTs/vo6MAefSjIzK6ksgbAI6Cupj6SO5C4S1xTUqQHGp+lLgPkREal8bLoLqQ/QF3hO0pGSugJIOhIYAazY/80xM7OWavIuo4hokDQJmAeUAQ9FxEpJU4HaiKgBHgRmSKojd2QwNq27UtIjwAtAAzAxInZK+igwJ3fdmXLg4Yj4xQHYPjMzy6jJQACIiLnA3IKyKXnTO4Axjaz7r8C/FpStBQY2t7NmZnbg+EllMzMDHAhmZpY4EMzMDHAgmJlZ4kAwMzPAgWBmZokDwczMAAeCmZklDgQzMwMcCGZmljgQzMwMcCCYmVniQDAzM8CBYGZmiQPBzMwAB4KZmSWZAkHSSEmrJdVJmlxkeSdJs9LyhZIq8pbdlMpXSzova5tmZta2mgwESWXAfcD5QD9gnKR+BdWuBrZGxEnAPcBdad1+5F6neSowEvi/ksoytmlmZm0oyxHCYKAuItZGxLvATGBUQZ1RwPQ0PRsYptwLk0cBMyPinYj4E1CX2svSppmZtaEs71TuBbySN18PDGmsTkQ0SNoG9Ejl/12wbq803VSbAEiaAExIs29KWp2hzwfSMcCm/WlAd7VST0rP++J93hfv8754X3vZFydkqZQlEFSkLDLWaay82JFJYZu5wogHgAf21cG2JKk2IqpL3Y/2wPvifd4X7/O+eN/Bti+ynDKqB47Lm+8NrG+sjqRyoDuwZR/rZmnTzMzaUJZAWAT0ldRHUkdyF4lrCurUAOPT9CXA/IiIVD423YXUB+gLPJexTTMza0NNnjJK1wQmAfOAMuChiFgpaSpQGxE1wIPADEl15I4MxqZ1V0p6BHgBaAAmRsROgGJttv7mHRDt5vRVO+B98T7vi/d5X7zvoNoXyv0hb2Zmhzs/qWxmZoADwczMEgdCRpIekvS6pBWl7kupSeos6TlJyyStlPT1UvepLRX7vyDpI5J+JemP6evRpexjW5B0nKRfS1qV/h98OZUfdvsCGv+5SDfPLEz7Y1a6kaZdciBk931yw28YvAOcGxEDgSpgpKQzStyntvR99v6/MBl4KiL6Ak+l+UNdA3BDRJwCnAFMTEPQHI77Ahr/ubgLuCftj63khvpplxwIGUXEAnJ3UB32IufNNNshfQ6buxMa+b+QP3zLdOCiNu1UCUTEhoj4fZreDqwiNxLBYbcvYJ8/F+eSG9IH2vn+cCBYi6RBCpcCrwO/ioiFpe5TiX00IjZA7hclcGyJ+9Om0gjHg4CFHMb7ovDnAlgDvBERDalK/vA97Y4DwVokInZGRBW5p8wHS+pf6j5ZaUj6MPAocH1E/KXU/Smlwp8L4JRi1dq2V9k5EGy/RMQbwNP4+sprkj4GkL6+XuL+tAlJHciFwY8i4qep+LDcF/nyfi7OAI5KQ/pAOx+mx4FgzSapp6Sj0nQX4DPAi6XtVcnlD98yHvhZCfvSJtIQ9w8CqyLi3/IWHXb7Ahr9uVgF/JrckD7QzveHn1TOSNKPgU+TG872NeCWiHiwpJ0qEUmV5C6OlZH7o+KRiJha2l61nWL/F4D/BzwCHA+8DIyJiEP6JgRJZwG/BZ4HdqXir5K7jnBY7Qto/OdC0ifIvfPlI8AS4H9FxDul62njHAhmZgb4lJGZmSUOBDMzAxwIZmaWOBDMzAxwIJiZWeJAMCsg6ShJXyp1P8zamgPBbG9HAQ4EO+w0+U5ls8PQncCJaZCyX6Wy88mNQXN7RMyS9GlgKrAZ+CSwAPhSROzKb0jSlcCFwIeAE4E5EfHPbbERZs3lIwSzvU0G1qRByv6b3Nj2A8kNRfCt3eP0kBu87AZgALlf9n/fSHtVwKWp3qWSjjuAfTdrMQeC2b6dBfw4jWL5GvAb4PS07LmIWBsRO4Efp7rFPBUR2yJiB/ACcMIB77VZCzgQzPZN+1hWOO5LSLpY0tL0qU7l+ePW7MSnaq2dciCY7W070DVNLyB3mqdMUk9gKPBcWjY4vS/3CHKnhH4XEXMioip9atu+62Yt50AwKxARm4FnJK0A/gZYDiwD5gP/HBF/TlWfJXcBegXwJ2BOCbpr1mo82qlZC6S7jG6MiAtK3Rez1uIjBDMzA3yEYGZmiY8QzMwMcCCYmVniQDAzM8CBYGZmiQPBzMwA+P/yhpcnQZifXQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "report_results('topn', result_cold, target_metric);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEWCAYAAACKSkfIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAGkdJREFUeJzt3X9wVeW97/H3hwCCLaJF6D0VMYjcKoYQvAE8o4IjFbF6RQ8ygr1VGa2eFry1Rz0Xe1pQdKy29tA6cMfa6tRja8FivaX+KFOllpaxaCg/BJE2UNRUqhh+iFXUwPf+sRcxbBOzE0J2wvN5zWRY61nP2vu7F8knK89e69mKCMzMLA1dil2AmZm1H4e+mVlCHPpmZglx6JuZJcShb2aWEIe+mVlCHPpmZglx6NshTdJPJd2f1zZGUq2kf/qY/bpKCkmvSSpp0N4927fuYNZtdrA49O1Q97+Bz0s6G0BSD+CHwPURsaWA/XcB4xqsnw+82eZVmrUTh74d0iKiFrgWuFfSJ4BZwMaI+LGkf5b0R0k7JG2RdLekbnkP8SBwWYP1y4D/athBUo2kMxus3ybpx9ny4ZIeyv462CHpOUlHt/0rNSuMQ98OeRHxc2AF8DPgauCabFMd8FXgaOA0YHyDbfv8AjhL0hGS+gCnAo+14OmnAocD/YE+wFeA3a17JWYHrmuxCzBrJ9OAjcB/RMQrABHxfIPtmyTdC4wB5jZofwd4EpgE9AQeBd5rwfN+QO6XygkR8QJQ1epXYNYGHPqWhIh4XdKbwLp9bZJOBL4L/A9yZ+NdgeWN7P5f5IaFepL7y6Alfgx8BnhY0hHkhou+ERF+I9iKwsM7lrIfAGvJnYUfAcwE1Ei/3wLHAUdGxLONbP8HuV8a+/y3fQsR8X5E3BwRJwGnAxcBX2ij+s1azKFvKesF7AT+IekkPjqeD0Dk5h8/H7iwicdZBUzOLvMcCfzLvg2SzpJUJqkL8Ba54Z49bfgazFrEoW8pux64nNxlmT8AFjTVMSLWRsSLTWz+D+BEYAfwTeChBts+Q+7N4LfIDS09Re4NZbOikD9ExcwsHT7TNzNLiEPfzCwhBYW+pPGSNkiqljSjke2HSVqQbV8uqTRrL5X0rqRV2dc9bVu+mZm1RLPX6WeTTc0DzgZqgOclLcp7U+tKYHtEnCBpMnAncEm2bWNEVLRx3WZm1gqF3Jw1EqiOiE0AkuYDE4CGoT8BuDlbXgjMldTY9c7NOvroo6O0tLQ1u5qZJWvFihVvRkTf5voVEvrHAK82WK8BRjXVJyLqJO0kN88IwEBJK8ldsvaNiPh9/hNIuprcnCgMGDCAqirfqW5m1hKSXi6kXyFj+o2dsedf59lUny3AgIgYDvwb8FB2K/r+HSPujYjKiKjs27fZX1RmZtZKhYR+DXBsg/X+wGtN9ZHUFegNbIuI97KpbYmIFeQmvPrvB1q0mZm1TiGh/zwwWNJASd2BycCivD6LyN3ZCHAxsCQiQlLffZ86JOl4YDCwqW1KNzOzlmp2TD8bo58OLAZKgPsjYp2k2UBVRCwC7gMelFQNbCP3iwFgNDA7+2i5PcC/RsS2lhb5wQcfUFNTw+7dnoa8o+nRowf9+/enW7f8zx4xs46ow03DUFlZGflv5P71r3+lV69e9OnTh1ZeFGQHQURQW1vLrl27GDhwYLHLMUuapBURUdlcv05xR+7u3bsd+B2QJPr06eO/wMw6kU4R+oADv4Py/4tZ59JpQt/MzA6cQ78DKC0t5c033zzgPmZmzfFn5JpZ0kpnPF7sEth8x3nt9lw+02+lzZs3c+KJJ3LVVVdRVlbGF77wBZ566ilOO+00Bg8ezHPPPce2bdu48MILKS8v59RTT2XNmjUA1NbWMm7cOIYPH84111xDwyuofvKTnzBy5EgqKiq45ppr2LPHn6xnZm3HoX8Aqqur+epXv8qaNWt46aWXeOihh/jDH/7AXXfdxe23386sWbMYPnw4a9as4fbbb+eyyy4D4JZbbuH0009n5cqVXHDBBbzyyisArF+/ngULFrBs2TJWrVpFSUkJP/3pT4v5Es3sEOPhnQMwcOBAhg4dCsDJJ5/M2LFjkcTQoUPZvHkzL7/8Mo888ggAZ511FrW1tezcuZOlS5fyi1/8AoDzzjuPo446CoCnn36aFStWMGLECADeffdd+vXrV4RXZmaHKof+ATjssMPql7t06VK/3qVLF+rq6uja9aOHd98ljo1d6hgRXH755XzrW986SBWbWeo8vHMQjR49un545plnnuHoo4/miCOO2K/9ySefZPv27QCMHTuWhQsX8sYbbwCwbds2Xn65oNlSzcwK4jP9g+jmm29m6tSplJeXc/jhh/PAAw8AMGvWLKZMmcIpp5zCmDFjGDBgAABDhgzhtttuY9y4cezdu5du3boxb948jjvuuGK+DDM7hHSKuXfWr1/PSSedVKSKrDn+/7HO7FC5ZPOQmnvHzMzahkPfzCwhDn0zs4Q49M3MEuLQNzNLiEPfzCwhnfI6/ba+xKo9Z7gzMysmn+kXqKSkhIqKCsrKypg0aRLvvPNOmzzu66+/zvnnn8+wYcMYMmQIn//854HcvD4bNmzYr+91113Ht7/9bZ555hl69+7N8OHD+exnP8vo0aN57LHHmn2ulStXctVVV7W4xhtuuIElS5a0eD8z63gc+gXq2bMnq1atYu3atXTv3p177rmn4H3r6uqa3DZz5kzOPvtsVq9ezYsvvsgdd9wBwOTJk5k/f359v71797Jw4UIuueQSAM444wxWrlzJhg0buPvuu5k+fTpPP/30x9Zx++23c+211xZc9z7XXnttfV1m1rk59FvhjDPOoLq6ms2bN1NWVlbfftddd3HzzTcDcOaZZ/L1r3+dMWPG8P3vf5+tW7cyceJERowYwYgRI1i2bBkAW7ZsoX///vWPUV5eDsCUKVP2C/2lS5dSWlra6JQMFRUVzJw5k7lz5wLw85//nLKyMoYNG8bo0aMB2LVrF2vWrGHYsGEAvP3220ydOpWhQ4dSXl7OI488wp49e7jiiisoKytj6NChzJkzB4DjjjuO2tpa/v73v7fVITSzIumUY/rFVFdXx5NPPsn48eOb7btjxw5+97vfAXDppZfyta99jdNPP51XXnmFc845h/Xr1zNt2jQuueQS5s6dy+c+9zmmTp3KZz7zGcrLy+nSpQurV69m2LBhzJ8/nylTpjT5XKeccgrf+c53AJg9ezaLFy/mmGOOYceOHQBUVVXt9wvq1ltvpXfv3rzwwgsAbN++nVWrVvG3v/2NtWvX1tff8PGXLVvGxIkTW3jEzKwj8Zl+gd59910qKiqorKxkwIABXHnllc3us28oBuCpp55i+vTpVFRUcMEFF/DWW2+xa9cuzjnnHDZt2sSXvvQlXnrpJYYPH87WrVuBD8/26+rq+OUvf8mkSZOafK6GcyiddtppXHHFFfzwhz+s/+StLVu20Ldv3/3qmTZtWv36UUcdxfHHH8+mTZu49tpr+fWvf80RRxxRv71fv3689tprBRwpM+vIfKZfoH1j+g117dqVvXv31q/v3r17v+2f+MQn6pf37t3Ls88+S8+ePT/y2J/61Ke49NJLufTSSzn//PNZunQpEydOZMqUKYwbN44xY8ZQXl7+sR+osnLlyvpJz+655x6WL1/O448/TkVFBatWraJnz5771RcRH5nT/6ijjmL16tUsXryYefPm8fDDD3P//ffXv7bGajezzqVThn5HucTy05/+NG+88Qa1tbV88pOf5LHHHmty2GfcuHHMnTuXG2+8EYBVq1ZRUVHBkiVLOPXUUzn88MPZtWsXGzdurJ9qedCgQfTp04cZM2Zw3XXXNVnHmjVruPXWW/nRj34EwMaNGxk1ahSjRo3iV7/6Fa+++ionnXQS3/3udz9Sz/e+9z0gN7yzZ88eunfvzsSJExk0aBBXXHFFff8///nPH/uXhpl1Dp0y9DuKbt26MXPmTEaNGsXAgQM58cQTm+x79913M23aNMrLy6mrq2P06NHcc889rFixgunTp9f/1XDVVVfVf1wi5IZ4brrpJi666KL9Hu/3v/89w4cP55133qFfv37cfffdjB07FoAbb7yRv/zlL0QEY8eOZdiwYUhi586d7Nq1i169evGNb3yDadOmUVZWRklJCbNmzWLQoEFMnTq1/q+XfZ/g9cEHH1BdXU1lZbOztppZB+f59BMyZ84cevXq1eJr9R999FH+9Kc/ceuttza63f8/1pl5Pn07ZH35y1/e73N9C1VXV8f1119/ECoys/bWaYZ3Gnvj0VqmR48efPGLX2zxfoVeNWRmHV+nONPv0aMHtbW1DpgOJiKora2lR48exS7FzArUKc70+/fvT01NTf3169Zx9OjRY787is2sY+sUod+tWzcGDhxY7DLMzDq9TjG8Y2ZmbcOhb2aWkIJCX9J4SRskVUua0cj2wyQtyLYvl1Sat32ApLcl3dA2ZZuZWWs0G/qSSoB5wLnAEGCKpCF53a4EtkfECcAc4M687XOAJw+8XDMzOxCFnOmPBKojYlNEvA/MBybk9ZkAPJAtLwTGKruoXtKFwCZgXduUbGZmrVVI6B8DvNpgvSZra7RPRNQBO4E+kj4B/B/glo97AklXS6qSVOXLMs3MDp5CQr+x22Dz75Jqqs8twJyIePvjniAi7o2IyoiobDjnu5mZta1CrtOvAY5tsN4fyP80jX19aiR1BXoD24BRwMWSvg0cCeyVtDsi5h5w5WZm1mKFhP7zwGBJA4G/AZOBS/P6LAIuB54FLgaWRG7OhDP2dZB0M/C2A9/MrHiaDf2IqJM0HVgMlAD3R8Q6SbOBqohYBNwHPCipmtwZ/uSDWbSZmbVOQdMwRMQTwBN5bTMbLO8GPvZjlSLi5lbUZ2Zmbch35JqZJcShb2aWEIe+mVlCHPpmZglx6JuZJcShb2aWEIe+mVlCHPpmZglx6JuZJcShb2aWEIe+mVlCHPpmZglx6JuZJcShb2aWEIe+mVlCHPpmZglx6JuZJcShb2aWEIe+mVlCHPpmZglx6JuZJcShb2aWEIe+mVlCHPpmZglx6JuZJcShb2aWkK7FLsDM2l/pjMeLXQKb7ziv2CUkyWf6ZmYJceibmSXEoW9mlhCHvplZQhz6ZmYJceibmSXEoW9mlpCCQl/SeEkbJFVLmtHI9sMkLci2L5dUmrWPlLQq+1ot6aK2Ld/MzFqi2dCXVALMA84FhgBTJA3J63YlsD0iTgDmAHdm7WuByoioAMYDP5DkG8LMzIqkkDP9kUB1RGyKiPeB+cCEvD4TgAey5YXAWEmKiHcioi5r7wFEWxRtZmatU0joHwO82mC9JmtrtE8W8juBPgCSRklaB7wA/GuDXwL1JF0tqUpS1datW1v+KszMrCCFhL4aacs/Y2+yT0Qsj4iTgRHATZJ6fKRjxL0RURkRlX379i2gJDMza41CQr8GOLbBen/gtab6ZGP2vYFtDTtExHrgH0BZa4s1M7MDU0joPw8MljRQUndgMrAor88i4PJs+WJgSUREtk9XAEnHAZ8FNrdJ5WZm1mLNXkkTEXWSpgOLgRLg/ohYJ2k2UBURi4D7gAclVZM7w5+c7X46MEPSB8Be4CsR8ebBeCFmZta8gi6fjIgngCfy2mY2WN4NTGpkvweBBw+wRjMzayO+I9fMLCEOfTOzhDj0zcwS4tA3M0uIQ9/MLCEOfTOzhDj0zcwS4tA3M0uIQ9/MLCEOfTOzhDj0zcwS4tA3M0uIQ9/MLCEOfTOzhDj0zcwS4tA3M0uIQ9/MLCEOfTOzhDj0zcwS4tA3M0uIQ9/MLCEOfTOzhDj0zcwS4tA3M0uIQ9/MLCEOfTOzhDj0zcwS4tA3M0uIQ9/MLCEOfTOzhDj0zcwS4tA3M0uIQ9/MLCEOfTOzhBQU+pLGS9ogqVrSjEa2HyZpQbZ9uaTSrP1sSSskvZD9e1bblm9mZi3RbOhLKgHmAecCQ4ApkobkdbsS2B4RJwBzgDuz9jeB/xkRQ4HLgQfbqnAzM2u5Qs70RwLVEbEpIt4H5gMT8vpMAB7IlhcCYyUpIlZGxGtZ+zqgh6TD2qJwMzNruUJC/xjg1QbrNVlbo30iog7YCfTJ6zMRWBkR7+U/gaSrJVVJqtq6dWuhtZuZWQsVEvpqpC1a0kfSyeSGfK5p7Aki4t6IqIyIyr59+xZQkpmZtUYhoV8DHNtgvT/wWlN9JHUFegPbsvX+wKPAZRGx8UALNjOz1isk9J8HBksaKKk7MBlYlNdnEbk3agEuBpZEREg6EngcuCkilrVV0WZm1jrNhn42Rj8dWAysBx6OiHWSZku6IOt2H9BHUjXwb8C+yzqnAycA35S0Kvvq1+avwszMCtK1kE4R8QTwRF7bzAbLu4FJjex3G3DbAdZoZmZtxHfkmpklxKFvZpYQh76ZWUIc+mZmCXHom5klxKFvZpaQgi7ZNDsUlM54vNglsPmO84pdgiXOZ/pmZglx6JuZJcShb2aWEIe+mVlCHPpmZglx6JuZJcShb2aWEIe+mVlCHPpmZglx6JuZJcShb2aWEIe+mVlCHPpmZglx6JuZJcShb2aWEIe+mVlCHPpmZglx6JuZJcShb2aWEIe+mVlCHPpmZglx6JuZJcShb2aWEIe+mVlCHPpmZglx6JuZJcShb2aWkIJCX9J4SRskVUua0cj2wyQtyLYvl1SatfeR9FtJb0ua27alm5lZSzUb+pJKgHnAucAQYIqkIXndrgS2R8QJwBzgzqx9N/BN4IY2q9jMzFqtkDP9kUB1RGyKiPeB+cCEvD4TgAey5YXAWEmKiH9ExB/Ihb+ZmRVZIaF/DPBqg/WarK3RPhFRB+wE+rRFgWZm1nYKCX010hat6NP0E0hXS6qSVLV169ZCdzMzsxYqJPRrgGMbrPcHXmuqj6SuQG9gW6FFRMS9EVEZEZV9+/YtdDczM2uhQkL/eWCwpIGSugOTgUV5fRYBl2fLFwNLIqLgM30zM2sfXZvrEBF1kqYDi4ES4P6IWCdpNlAVEYuA+4AHJVWTO8OfvG9/SZuBI4Duki4ExkXEi23/UszMrDnNhj5ARDwBPJHXNrPB8m5gUhP7lh5AfWZm1oZ8R66ZWUIc+mZmCXHom5klpKAxfeu8Smc8XuwSANh8x3nFLsHM8Jm+mVlSHPpmZglx6JuZJcShb2aWEIe+mVlCHPpmZglx6JuZJcShb2aWEIe+mVlCHPpmZglx6JuZJcShb2aWkENywjVPMmZm1jif6ZuZJcShb2aWEIe+mVlCHPpmZglx6JuZJcShb2aWEIe+mVlCHPpmZglx6JuZJcShb2aWEIe+mVlCHPpmZglx6JuZJcShb2aWEIe+mVlCHPpmZglx6JuZJcShb2aWkIJCX9J4SRskVUua0cj2wyQtyLYvl1TaYNtNWfsGSee0XelmZtZSzYa+pBJgHnAuMASYImlIXrcrge0RcQIwB7gz23cIMBk4GRgP/N/s8czMrAgKOdMfCVRHxKaIeB+YD0zI6zMBeCBbXgiMlaSsfX5EvBcRfwWqs8czM7Mi6FpAn2OAVxus1wCjmuoTEXWSdgJ9svY/5u17TP4TSLoauDpbfVvShoKqP7iOBt48kAfQnW1USfH5WHzogI7FIXQcwMeioY5wLI4rpFMhoa9G2qLAPoXsS0TcC9xbQC3tRlJVRFQWu46OwMfiQz4WH/Kx+FBnOhaFDO/UAMc2WO8PvNZUH0ldgd7AtgL3NTOzdlJI6D8PDJY0UFJ3cm/MLsrrswi4PFu+GFgSEZG1T86u7hkIDAaea5vSzcyspZod3snG6KcDi4ES4P6IWCdpNlAVEYuA+4AHJVWTO8OfnO27TtLDwItAHTAtIvYcpNfS1jrUcFOR+Vh8yMfiQz4WH+o0x0K5E3IzM0uB78g1M0uIQ9/MLCEO/TyS7pf0hqS1xa6lmCT1kPScpNWS1km6pdg1tbfGvhckfUrSbyT9Jfv3qGLW2B4kHSvpt5LWZ98LX83aUzwWjf5cZBe6LM+OxYLsopcOyaH/UT8mN2VE6t4DzoqIYUAFMF7SqUWuqb39mI9+L8wAno6IwcDT2fqhrg64PiJOAk4FpmVTrKR4LJr6ubgTmJMdi+3kpqbpkBz6eSJiKbkrkJIWOW9nq92yr6Te9W/ie6HhlCMPABe2a1FFEBFbIuJP2fIuYD25O+tTPBZN/VycRW4KGujgx8Khb02SVCJpFfAG8JuIWF7smjqAT0fEFsiFIdCvyPW0q2wG3eHAchI9Fvk/F8BGYEdE1GVdGp1upqNw6FuTImJPRFSQu5N6pKSyYtdkxSPpk8AjwHUR8Vax6ymW/J8L4KTGurVvVYVz6FuzImIH8Ax+rwPgdUn/BJD9+0aR62kXkrqRC/yfRsQvsuYkj8U+DX4uTgWOzKaggQ4+3YxD3xolqa+kI7PlnsDngJeKW1WH0HDKkcuBXxaxlnaRTZN+H7A+Iv6zwaYUj0VjPxfrgd+Sm4IGOvix8B25eST9DDiT3FSprwOzIuK+ohZVBJLKyb0hVULu5ODhiJhd3KraV2PfC8D/Ax4GBgCvAJMi4pB+41/S6cDvgReAvVnz18mN66d2LBr9uZB0PLnPGvkUsBL4XxHxXvEqbZpD38wsIR7eMTNLiEPfzCwhDn0zs4Q49M3MEuLQNzNLiEPfkiXpSElfKXYdZu3JoW8pOxJw6FtSmv2MXLND2B3AoGzyrN9kbeeSmzfltohYIOlMYDZQC3wWWAp8JSL2NnwgSVcAFwCHA4OARyPi39vjRZi1hM/0LWUzgI3Z5Fl/JDc/+jByt9Z/Z9+8MuQm1boeGEou0P+licerAC7J+l0i6diDWLtZqzj0zXJOB36WzaD4OvA7YES27bmI2BQRe4CfZX0b83RE7IyI3cCLwHEHvWqzFnLom+XoY7blz1USki6StCr7qszaG861sgcPn1oH5NC3lO0CemXLS8kNyZRI6guMBp7Lto3MPgO1C7nhmz9ExKMRUZF9VbV/6Wat49C3ZEVELbAs++DzfwbWAKuBJcC/R8Tfs67PknvTdy3wV+DRIpRr1iY8y6bZx8iu3rkhIs4vdi1mbcFn+mZmCfGZvplZQnymb2aWEIe+mVlCHPpmZglx6JuZJcShb2aWkP8PQ5PXCCPJLtEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "report_results('topn', result_cold, 'coverage');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### saving data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_results(experiment_name+'_coldstart', cv=result_cold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}