HighLevelFeedbackStats.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "68bf00c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import json\n",
    "from scipy.stats import fisher_exact"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "235f806b",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "22da247d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "infile = open(\"interaction_data.pkl\",'rb')\n",
    "d = pickle.load(infile)\n",
    "infile.close()\n",
    "\n",
    "infile = open(\"phase3_selections.pkl\",'rb')\n",
    "selected_item_interactions = pickle.load(infile)\n",
    "infile.close()\n",
    "\n",
    "infile = open(\"df_completed_participation.pkl\",'rb')\n",
    "df_completed_participation = pickle.load(infile)\n",
    "infile.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4bee11f",
   "metadata": {},
   "source": [
    "# High-level feedback: relative comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "aa978a25",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = d.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1bedf510",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-4-4084f8a5de2b>:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  relative_comparison_data[\"relative_comparison\"] = 0\n",
      "<ipython-input-4-4084f8a5de2b>:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  relative_comparison_data[\"relative_comparison\"] = relative_comparison_data.data.map(lambda x: json.loads(x).get(\"value\", 0))\n",
      "<ipython-input-4-4084f8a5de2b>:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  relative_comparison_data[\"relative_comparison_numeric\"] = relative_comparison_data[\"relative_comparison\"].map(lambda x: dct[x])\n"
     ]
    }
   ],
   "source": [
    "relative_comparison_data = df[df.data.str.contains(\"radio-sub-component\")]\n",
    "relative_comparison_data[\"relative_comparison\"] = 0\n",
    "\n",
    "relative_comparison_data[\"relative_comparison\"] = relative_comparison_data.data.map(lambda x: json.loads(x).get(\"value\", 0))\n",
    "dct = {\"first\":-2, \"second\":-1,\"third\":0,\"fourth\":1,\"fifth\":2}\n",
    "relative_comparison_data[\"relative_comparison_numeric\"] = relative_comparison_data[\"relative_comparison\"].map(lambda x: dct[x])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "611d04a8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1063, 12)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "relative_comparison_data = relative_comparison_data.sort_values([\"participation\",\"iteration\",\"time\"])\n",
    "relative_comparison_data = relative_comparison_data.drop_duplicates([\"participation\",\"iteration\",\"variant\"],keep=\"last\")\n",
    "relative_comparison_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d6327cd2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>relative_comparison</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>relative_comparison_numeric</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>-2</th>\n",
       "      <td>121</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-1</th>\n",
       "      <td>261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>106</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             relative_comparison\n",
       "relative_comparison_numeric                     \n",
       "-2                                           121\n",
       "-1                                           261\n",
       " 0                                           316\n",
       " 1                                           259\n",
       " 2                                           106"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rcgb = relative_comparison_data.groupby(\"relative_comparison_numeric\").count()[[\"relative_comparison\"]]\n",
    "rcgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2243d073",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>relative_comparison_numeric</th>\n",
       "      <th>-2</th>\n",
       "      <th>-1</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>result_layout</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>column-single</th>\n",
       "      <td>23</td>\n",
       "      <td>56</td>\n",
       "      <td>55</td>\n",
       "      <td>47</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>columns</th>\n",
       "      <td>21</td>\n",
       "      <td>48</td>\n",
       "      <td>50</td>\n",
       "      <td>51</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max-columns</th>\n",
       "      <td>22</td>\n",
       "      <td>30</td>\n",
       "      <td>52</td>\n",
       "      <td>40</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row-single</th>\n",
       "      <td>19</td>\n",
       "      <td>40</td>\n",
       "      <td>65</td>\n",
       "      <td>33</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row-single-scrollable</th>\n",
       "      <td>18</td>\n",
       "      <td>32</td>\n",
       "      <td>49</td>\n",
       "      <td>43</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rows</th>\n",
       "      <td>18</td>\n",
       "      <td>55</td>\n",
       "      <td>45</td>\n",
       "      <td>45</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "relative_comparison_numeric  -2  -1   0   1   2\n",
       "result_layout                                  \n",
       "column-single                23  56  55  47  17\n",
       "columns                      21  48  50  51  13\n",
       "max-columns                  22  30  52  40  28\n",
       "row-single                   19  40  65  33  16\n",
       "row-single-scrollable        18  32  49  43  15\n",
       "rows                         18  55  45  45  17"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contingency_table_ind = pd.crosstab(relative_comparison_data.result_layout,relative_comparison_data.relative_comparison_numeric)\n",
    "contingency_table_ind"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "adf227d6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>relative_comparison_numeric</th>\n",
       "      <th>-2</th>\n",
       "      <th>-1</th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>layout_high_level</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cols</th>\n",
       "      <td>66</td>\n",
       "      <td>134</td>\n",
       "      <td>157</td>\n",
       "      <td>138</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rows</th>\n",
       "      <td>55</td>\n",
       "      <td>127</td>\n",
       "      <td>159</td>\n",
       "      <td>121</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "relative_comparison_numeric  -2   -1    0    1   2\n",
       "layout_high_level                                 \n",
       "cols                         66  134  157  138  58\n",
       "rows                         55  127  159  121  48"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contingency_table = pd.crosstab(relative_comparison_data.layout_high_level,relative_comparison_data.relative_comparison_numeric)\n",
    "contingency_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "32a2bd83",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.2609490440937195\n"
     ]
    }
   ],
   "source": [
    "from scipy.stats import binom_test, fisher_exact\n",
    "succ = contingency_table.loc[\"rows\",-2]+contingency_table.loc[\"rows\",-1]\n",
    "fail = contingency_table.loc[\"rows\",1]+contingency_table.loc[\"rows\",2]\n",
    "print(binom_test(succ, succ+fail,p=0.5,alternative='greater'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "771112bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.27915071070263264\n"
     ]
    }
   ],
   "source": [
    "from scipy.stats import binom_test, fisher_exact\n",
    "rcgb = relative_comparison_data.groupby(\"relative_comparison_numeric\").count()[[\"relative_comparison\"]]\n",
    "succ = rcgb.loc[-2,\"relative_comparison\"]+rcgb.loc[-1,\"relative_comparison\"]\n",
    "fail = rcgb.loc[1,\"relative_comparison\"]+rcgb.loc[2,\"relative_comparison\"]\n",
    "print(binom_test(succ, succ+fail,p=0.5,alternative='greater'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9cbea920",
   "metadata": {},
   "source": [
    "- For relative comparison, advantaged algorithm received slightly higher preferences - but not stat. sign even while not distingushing among individual layouts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "975f7011",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-11-1178ec4d733f>:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  relative_comp_no_eq[\"winning\"] = \"first\"\n",
      "C:\\Users\\lpesk\\AppData\\Roaming\\Python\\Python38\\site-packages\\pandas\\core\\indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>winning</th>\n",
       "      <th>first</th>\n",
       "      <th>second</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>layout_high_level</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>cols</th>\n",
       "      <td>200</td>\n",
       "      <td>196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rows</th>\n",
       "      <td>182</td>\n",
       "      <td>169</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "winning            first  second\n",
       "layout_high_level               \n",
       "cols                 200     196\n",
       "rows                 182     169"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "relative_comp_no_eq = relative_comparison_data.loc[relative_comparison_data.relative_comparison_numeric != 0]\n",
    "relative_comp_no_eq[\"winning\"] = \"first\"\n",
    "relative_comp_no_eq.loc[relative_comp_no_eq.relative_comparison_numeric > 0, \"winning\"] = \"second\"\n",
    "\n",
    "contingency_table = pd.crosstab(relative_comp_no_eq.layout_high_level,relative_comp_no_eq.winning)\n",
    "contingency_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d82511e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.9475218658892128, 0.714631698077167)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy.stats import fisher_exact\n",
    "fisher_exact(contingency_table)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fe06f1bb",
   "metadata": {},
   "source": [
    "- seemingly, no difference here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "aeba3a9c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>relative_comparison_numeric</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>result_layout</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>column-single</th>\n",
       "      <td>-0.106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>columns</th>\n",
       "      <td>-0.071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max-columns</th>\n",
       "      <td>0.128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row-single</th>\n",
       "      <td>-0.075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row-single-scrollable</th>\n",
       "      <td>0.032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rows</th>\n",
       "      <td>-0.067</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       relative_comparison_numeric\n",
       "result_layout                                     \n",
       "column-single                               -0.106\n",
       "columns                                     -0.071\n",
       "max-columns                                  0.128\n",
       "row-single                                  -0.075\n",
       "row-single-scrollable                        0.032\n",
       "rows                                        -0.067"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "relative_comparison_data.groupby([\"result_layout\"])[[\"relative_comparison_numeric\"]].mean().round(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "358e28d2",
   "metadata": {},
   "source": [
    "- not much difference here either"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b35f3f92",
   "metadata": {},
   "source": [
    "# High-level feedback: rating algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2c41eede",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-14-a65bcddd8f81>:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  rating_data[\"variant\"] = rating_data.data.map(lambda x: json.loads(x).get(\"variant\", -1))\n",
      "<ipython-input-14-a65bcddd8f81>:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  rating_data[\"rating\"] = rating_data.data.map(lambda x: json.loads(x).get(\"new_rating\", -1))\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>participation</th>\n",
       "      <th>interaction_type</th>\n",
       "      <th>time</th>\n",
       "      <th>data</th>\n",
       "      <th>iteration</th>\n",
       "      <th>result_layout</th>\n",
       "      <th>GAMMA</th>\n",
       "      <th>DELTA</th>\n",
       "      <th>variant</th>\n",
       "      <th>layout_high_level</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3287</th>\n",
       "      <td>36</td>\n",
       "      <td>on-input</td>\n",
       "      <td>2023-01-15 23:16:21.116547</td>\n",
       "      <td>{\"variant\": 0, \"variant_name\": \"DELTA\", \"old_r...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>row-single-scrollable</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>rows</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3288</th>\n",
       "      <td>36</td>\n",
       "      <td>on-input</td>\n",
       "      <td>2023-01-15 23:16:22.800032</td>\n",
       "      <td>{\"variant\": 1, \"variant_name\": \"GAMMA\", \"old_r...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>row-single-scrollable</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>rows</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3307</th>\n",
       "      <td>36</td>\n",
       "      <td>on-input</td>\n",
       "      <td>2023-01-15 23:16:50.703590</td>\n",
       "      <td>{\"variant\": 0, \"variant_name\": \"GAMMA\", \"old_r...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>columns</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>cols</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3306</th>\n",
       "      <td>36</td>\n",
       "      <td>on-input</td>\n",
       "      <td>2023-01-15 23:16:49.672394</td>\n",
       "      <td>{\"variant\": 1, \"variant_name\": \"DELTA\", \"old_r...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>columns</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>cols</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3316</th>\n",
       "      <td>36</td>\n",
       "      <td>on-input</td>\n",
       "      <td>2023-01-15 23:17:18.883702</td>\n",
       "      <td>{\"variant\": 0, \"variant_name\": \"GAMMA\", \"old_r...</td>\n",
       "      <td>3.0</td>\n",
       "      <td>max-columns</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>cols</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47742</th>\n",
       "      <td>261</td>\n",
       "      <td>on-input</td>\n",
       "      <td>2023-02-05 18:45:48.779783</td>\n",
       "      <td>{\"variant\": 1, \"variant_name\": \"DELTA\", \"old_r...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>row-single-scrollable</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>rows</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47761</th>\n",
       "      <td>261</td>\n",
       "      <td>on-input</td>\n",
       "      <td>2023-02-05 18:46:24.734999</td>\n",
       "      <td>{\"variant\": 0, \"variant_name\": \"GAMMA\", \"old_r...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>columns</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>cols</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47773</th>\n",
       "      <td>261</td>\n",
       "      <td>on-input</td>\n",
       "      <td>2023-02-05 18:46:45.523096</td>\n",
       "      <td>{\"variant\": 1, \"variant_name\": \"DELTA\", \"old_r...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>columns</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>cols</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47789</th>\n",
       "      <td>261</td>\n",
       "      <td>on-input</td>\n",
       "      <td>2023-02-05 18:47:15.081066</td>\n",
       "      <td>{\"variant\": 0, \"variant_name\": \"GAMMA\", \"old_r...</td>\n",
       "      <td>8.0</td>\n",
       "      <td>max-columns</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>cols</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47795</th>\n",
       "      <td>261</td>\n",
       "      <td>on-input</td>\n",
       "      <td>2023-02-05 18:47:29.596114</td>\n",
       "      <td>{\"variant\": 1, \"variant_name\": \"DELTA\", \"old_r...</td>\n",
       "      <td>8.0</td>\n",
       "      <td>max-columns</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>cols</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2138 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       participation interaction_type                        time  \\\n",
       "id                                                                  \n",
       "3287              36         on-input  2023-01-15 23:16:21.116547   \n",
       "3288              36         on-input  2023-01-15 23:16:22.800032   \n",
       "3307              36         on-input  2023-01-15 23:16:50.703590   \n",
       "3306              36         on-input  2023-01-15 23:16:49.672394   \n",
       "3316              36         on-input  2023-01-15 23:17:18.883702   \n",
       "...              ...              ...                         ...   \n",
       "47742            261         on-input  2023-02-05 18:45:48.779783   \n",
       "47761            261         on-input  2023-02-05 18:46:24.734999   \n",
       "47773            261         on-input  2023-02-05 18:46:45.523096   \n",
       "47789            261         on-input  2023-02-05 18:47:15.081066   \n",
       "47795            261         on-input  2023-02-05 18:47:29.596114   \n",
       "\n",
       "                                                    data  iteration  \\\n",
       "id                                                                    \n",
       "3287   {\"variant\": 0, \"variant_name\": \"DELTA\", \"old_r...        1.0   \n",
       "3288   {\"variant\": 1, \"variant_name\": \"GAMMA\", \"old_r...        1.0   \n",
       "3307   {\"variant\": 0, \"variant_name\": \"GAMMA\", \"old_r...        2.0   \n",
       "3306   {\"variant\": 1, \"variant_name\": \"DELTA\", \"old_r...        2.0   \n",
       "3316   {\"variant\": 0, \"variant_name\": \"GAMMA\", \"old_r...        3.0   \n",
       "...                                                  ...        ...   \n",
       "47742  {\"variant\": 1, \"variant_name\": \"DELTA\", \"old_r...        6.0   \n",
       "47761  {\"variant\": 0, \"variant_name\": \"GAMMA\", \"old_r...        7.0   \n",
       "47773  {\"variant\": 1, \"variant_name\": \"DELTA\", \"old_r...        7.0   \n",
       "47789  {\"variant\": 0, \"variant_name\": \"GAMMA\", \"old_r...        8.0   \n",
       "47795  {\"variant\": 1, \"variant_name\": \"DELTA\", \"old_r...        8.0   \n",
       "\n",
       "               result_layout  GAMMA  DELTA  variant layout_high_level  rating  \n",
       "id                                                                             \n",
       "3287   row-single-scrollable    1.0    0.0        0              rows       4  \n",
       "3288   row-single-scrollable    1.0    0.0        1              rows       4  \n",
       "3307                 columns    0.0    1.0        0              cols       4  \n",
       "3306                 columns    0.0    1.0        1              cols       4  \n",
       "3316             max-columns    0.0    1.0        0              cols       2  \n",
       "...                      ...    ...    ...      ...               ...     ...  \n",
       "47742  row-single-scrollable    0.0    1.0        1              rows       4  \n",
       "47761                columns    0.0    1.0        0              cols       1  \n",
       "47773                columns    0.0    1.0        1              cols       3  \n",
       "47789            max-columns    0.0    1.0        0              cols       1  \n",
       "47795            max-columns    0.0    1.0        1              cols       1  \n",
       "\n",
       "[2138 rows x 11 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rating_data = df[df.data.str.contains('\"input_type\": \"rating\"')]\n",
    "rating_data[\"variant\"] = rating_data.data.map(lambda x: json.loads(x).get(\"variant\", -1))\n",
    "rating_data[\"rating\"] = rating_data.data.map(lambda x: json.loads(x).get(\"new_rating\", -1))\n",
    "rating_data = rating_data.sort_values([\"participation\",\"iteration\",\"variant\",\"time\"])\n",
    "rating_data = rating_data.drop_duplicates([\"participation\",\"iteration\",\"variant\"],keep=\"last\")\n",
    "rating_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "2e530437",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>variant</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3.123480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.081384</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           rating\n",
       "variant          \n",
       "0        3.123480\n",
       "1        3.081384"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rating_data.groupby(\"variant\")[[\"rating\"]].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "cad6f533",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ttest_relResult(statistic=0.9515673319215723, pvalue=0.34259955939177045)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy.stats import ttest_ind, ttest_rel\n",
    "dx = rating_data.loc[((rating_data.variant==0) & (rating_data.result_layout==\"rows\"))]\n",
    "dy = rating_data.loc[((rating_data.variant==1) & (rating_data.result_layout==\"rows\"))]\n",
    "dx = dx.set_index([\"participation\",\"iteration\"])\n",
    "dy = dy.set_index([\"participation\",\"iteration\"])\n",
    "dRes = dx.join(dy, how=\"inner\", rsuffix = \"_1\")\n",
    "ttest_rel(dRes[\"rating\"],dRes[\"rating_1\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6371b641",
   "metadata": {},
   "source": [
    "- not significant enough"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "1bb4800b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>result_layout</th>\n",
       "      <th>variant</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">column-single</th>\n",
       "      <th>0</th>\n",
       "      <td>3.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">columns</th>\n",
       "      <th>0</th>\n",
       "      <td>3.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">max-columns</th>\n",
       "      <th>0</th>\n",
       "      <td>3.12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">row-single</th>\n",
       "      <th>0</th>\n",
       "      <td>3.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">row-single-scrollable</th>\n",
       "      <th>0</th>\n",
       "      <td>3.12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">rows</th>\n",
       "      <th>0</th>\n",
       "      <td>3.24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               rating\n",
       "result_layout         variant        \n",
       "column-single         0          3.06\n",
       "                      1          2.99\n",
       "columns               0          3.14\n",
       "                      1          3.04\n",
       "max-columns           0          3.12\n",
       "                      1          3.14\n",
       "row-single            0          3.06\n",
       "                      1          3.07\n",
       "row-single-scrollable 0          3.12\n",
       "                      1          3.11\n",
       "rows                  0          3.24\n",
       "                      1          3.14"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rating_data.groupby([\"result_layout\",\"variant\"])[[\"rating\"]].mean().round(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2705281d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>layout_high_level</th>\n",
       "      <th>variant</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">cols</th>\n",
       "      <th>0</th>\n",
       "      <td>3.105735</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.055556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">rows</th>\n",
       "      <th>0</th>\n",
       "      <td>3.142857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.109589</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             rating\n",
       "layout_high_level variant          \n",
       "cols              0        3.105735\n",
       "                  1        3.055556\n",
       "rows              0        3.142857\n",
       "                  1        3.109589"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rating_data.groupby([\"layout_high_level\",\"variant\"])[[\"rating\"]].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f9fdcf6d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ttest_indResult(statistic=0.8032994394995778, pvalue=0.422084454468898)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dx = rating_data.loc[rating_data.result_layout==\"rows\",\"rating\"]\n",
    "dy = rating_data.loc[rating_data.result_layout==\"row-single-scrollable\",\"rating\"]\n",
    "ttest_ind(dx,dy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "2181bc52",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ttest_indResult(statistic=1.5811426599496048, pvalue=0.11399347602174799)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dx = rating_data.loc[rating_data.result_layout==\"rows\",\"rating\"]\n",
    "dy = rating_data.loc[rating_data.result_layout!=\"rows\",\"rating\"]\n",
    "ttest_ind(dx,dy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e34daa15",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ttest_relResult(statistic=0.9515673319215723, pvalue=0.34259955939177045)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dx = rating_data.loc[rating_data.result_layout==\"rows\"].loc[rating_data.variant==0, \"rating\"]\n",
    "dy = rating_data.loc[rating_data.result_layout==\"rows\"].loc[rating_data.variant==1, \"rating\"]\n",
    "ttest_rel(dx,dy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "f518bd9b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ttest_relResult(statistic=1.0916485541893324, pvalue=0.27639963088400127)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dx = rating_data.loc[rating_data.result_layout==\"columns\"].loc[rating_data.variant==0, \"rating\"]\n",
    "dy = rating_data.loc[rating_data.result_layout==\"columns\"].loc[rating_data.variant==1, \"rating\"]\n",
    "ttest_rel(dx,dy)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "12bfad51",
   "metadata": {},
   "source": [
    "### There seems to be some minor differences between rating patterns of individual layouts, but not significant enough"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c88e7704",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "8b08d538",
   "metadata": {},
   "source": [
    "# How well low-level feedback corresponds to the high-level ones"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "498c2c32",
   "metadata": {},
   "source": [
    "### Get statistics on items that were mutually recommended by both RS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "af9a6130",
   "metadata": {},
   "outputs": [],
   "source": [
    "def getMovieId(x):\n",
    "    return x[\"movie_id\"]\n",
    "\n",
    "itemsDict = {\"participation\":[],\n",
    "            \"iteration\":[],\n",
    "            \"layout_high_level\":[],\n",
    "            \"results_layout\":[],\n",
    "            \"order\":[], \n",
    "            \"algorithm\":[], \n",
    "            \"rank\":[], \n",
    "            \"movieID\":[] \n",
    "            }\n",
    "\n",
    "pageLoads = d.loc[d.interaction_type == \"loaded-page\"]\n",
    "\n",
    "for idx, row in pageLoads.iterrows():\n",
    "    dt = json.loads(row.data)\n",
    "    page = dt[\"page\"]\n",
    "    if page == \"compare_algorithms\":\n",
    "        deltaMovies = dt[\"context\"][\"extra\"][\"movies\"][\"delta\"][\"movies\"]\n",
    "        deltaOrder = dt[\"context\"][\"extra\"][\"movies\"][\"delta\"][\"order\"]\n",
    "        gammaMovies = dt[\"context\"][\"extra\"][\"movies\"][\"gamma\"][\"movies\"]\n",
    "        gammaOrder = dt[\"context\"][\"extra\"][\"movies\"][\"gamma\"][\"order\"]\n",
    "        \n",
    "        deltaMIDs = list(map(getMovieId, deltaMovies))\n",
    "        gammaMIDs = list(map(getMovieId, gammaMovies))\n",
    "        \n",
    "        for c, val in enumerate(deltaMIDs):\n",
    "            itemsDict[\"participation\"].append(row[\"participation\"])\n",
    "            itemsDict[\"iteration\"].append(row[\"iteration\"])\n",
    "            itemsDict[\"layout_high_level\"].append(row[\"layout_high_level\"])\n",
    "            itemsDict[\"results_layout\"].append(row[\"result_layout\"])\n",
    "            itemsDict[\"order\"].append(deltaOrder)\n",
    "            itemsDict[\"algorithm\"].append(\"delta\")\n",
    "            itemsDict[\"rank\"].append(c)\n",
    "            itemsDict[\"movieID\"].append(val)\n",
    "            \n",
    "        for c, val in enumerate(gammaMIDs):\n",
    "            itemsDict[\"participation\"].append(row[\"participation\"])\n",
    "            itemsDict[\"iteration\"].append(row[\"iteration\"])\n",
    "            itemsDict[\"layout_high_level\"].append(row[\"layout_high_level\"])\n",
    "            itemsDict[\"results_layout\"].append(row[\"result_layout\"])\n",
    "            itemsDict[\"order\"].append(gammaOrder)\n",
    "            itemsDict[\"algorithm\"].append(\"gamma\")\n",
    "            itemsDict[\"rank\"].append(c)\n",
    "            itemsDict[\"movieID\"].append(val)        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "9937dce5",
   "metadata": {},
   "outputs": [],
   "source": [
    "displayedItems = pd.DataFrame(itemsDict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "36569099",
   "metadata": {},
   "outputs": [],
   "source": [
    "displayedItemsLeft = displayedItems.loc[displayedItems.order == 0]\n",
    "displayedItemsLeft = displayedItemsLeft.set_index([\"participation\",\"iteration\",\"movieID\"])[[\"layout_high_level\",\"results_layout\",\"algorithm\",\"rank\"]]\n",
    "displayedItemsRight = displayedItems.loc[displayedItems.order == 1]\n",
    "displayedItemsRight = displayedItemsRight.set_index([\"participation\",\"iteration\",\"movieID\"])[[\"algorithm\",\"rank\"]]\n",
    "\n",
    "pairedItems = displayedItemsLeft.join(displayedItemsRight, how=\"inner\", rsuffix=\"_r\")\n",
    "pairedItems[\"leftRightRankComparison\"] = 0\n",
    "pairedItems.loc[pairedItems[\"rank\"] < pairedItems.rank_r, \"leftRightRankComparison\"] = -1\n",
    "pairedItems.loc[pairedItems[\"rank\"] > pairedItems.rank_r, \"leftRightRankComparison\"] = 1\n",
    "\n",
    "pairedItems[\"leftRightRelaxed\"] = 0\n",
    "pairedItems.loc[pairedItems[\"rank\"] - pairedItems.rank_r < -1, \"leftRightRelaxed\"] = -1\n",
    "pairedItems.loc[pairedItems[\"rank\"] - pairedItems.rank_r > 1, \"leftRightRelaxed\"] = 1\n",
    "\n",
    "pairedItems[\"rankDiff\"] = pairedItems[\"rank\"] - pairedItems[\"rank_r\"] # positive values: better ranking of the right one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a65b1823",
   "metadata": {},
   "outputs": [],
   "source": [
    "sIt = selected_item_interactions[[\"participation\",\"time\",\"iteration\",\"variant\",\"movieID\"]]\n",
    "sIt = sIt.set_index([\"participation\",\"iteration\",\"movieID\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "4e83126e",
   "metadata": {},
   "outputs": [],
   "source": [
    "pairedSelectedItems = pairedItems.join(sIt, how=\"inner\", rsuffix=\"_selected\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "add30f11",
   "metadata": {},
   "outputs": [],
   "source": [
    "rt1 = d.drop_duplicates([\"participation\",\"iteration\"]).set_index([\"participation\",\"iteration\"])[[\"result_layout\",\"layout_high_level\"]]\n",
    "rt1[\"variant\"] = 0\n",
    "rt2 = rt1.copy()\n",
    "rt2[\"variant\"] = 1\n",
    "resultTypes = pd.concat([rt1,rt2]).reset_index().set_index([\"participation\",\"iteration\",\"variant\"])\n",
    "#this is due to missing data in case of no items were selected from particular variant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "89739c36",
   "metadata": {},
   "outputs": [],
   "source": [
    "gb = selected_item_interactions.groupby([\"participation\",\"iteration\",\"variant\"])[[\"movieID\"]].count()\n",
    "gb.columns = [\"selectedCount\"]\n",
    "gb = resultTypes.join(gb, how=\"outer\")\n",
    "gb = gb.fillna(0)\n",
    "meanUser = gb.reset_index().groupby(\"participation\")[[\"selectedCount\"]].mean()\n",
    "meanUser.columns = [\"meanPerUser\"]\n",
    "gb = gb.join(meanUser)\n",
    "\n",
    "pairedRes = pairedSelectedItems.reset_index()\n",
    "pairedRes[\"variant\"] = (pairedRes[\"variant\"]+1).mod(2)\n",
    "pairedRes = pairedRes.groupby([\"participation\",\"iteration\",\"variant\"])[[\"movieID\"]].count()\n",
    "pairedRes.columns=[\"pairedSelectedMovies\"]\n",
    "gb = gb.join(pairedRes, how=\"outer\")\n",
    "gb.fillna(0, inplace=True)\n",
    "gb[\"selectedWithPaired\"] = gb[\"selectedCount\"] + gb[\"pairedSelectedMovies\"]\n",
    "\n",
    "meanUser = gb.reset_index().groupby(\"participation\")[[\"selectedWithPaired\"]].mean()\n",
    "meanUser.columns = [\"meanPerUserWithPaired\"]\n",
    "gb = gb.join(meanUser)\n",
    "gb[\"userNormSelected\"] = gb[\"selectedCount\"] - gb[\"meanPerUser\"]\n",
    "gb[\"userNormSelectedWithPaired\"] = gb[\"selectedWithPaired\"] - gb[\"meanPerUserWithPaired\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "a8fa6156",
   "metadata": {},
   "outputs": [],
   "source": [
    "r = rating_data.set_index([\"participation\",\"iteration\",\"variant\"])[[\"rating\"]]\n",
    "gb = gb.join(r, how=\"inner\")\n",
    "gbr = gb[[\"selectedCount\",\"selectedWithPaired\",\"userNormSelected\",\"userNormSelectedWithPaired\",\"rating\",\"result_layout\",\"layout_high_level\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "df29803b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "row-single-scrollable\n",
      "                    selectedCount  selectedWithPaired  rating\n",
      "selectedCount                1.00                0.93    0.62\n",
      "selectedWithPaired           0.93                1.00    0.61\n",
      "rating                       0.62                0.61    1.00\n",
      "\n",
      "columns\n",
      "                    selectedCount  selectedWithPaired  rating\n",
      "selectedCount                1.00                0.93    0.60\n",
      "selectedWithPaired           0.93                1.00    0.62\n",
      "rating                       0.60                0.62    1.00\n",
      "\n",
      "max-columns\n",
      "                    selectedCount  selectedWithPaired  rating\n",
      "selectedCount                1.00                0.95    0.63\n",
      "selectedWithPaired           0.95                1.00    0.65\n",
      "rating                       0.63                0.65    1.00\n",
      "\n",
      "rows\n",
      "                    selectedCount  selectedWithPaired  rating\n",
      "selectedCount                1.00                0.87    0.60\n",
      "selectedWithPaired           0.87                1.00    0.62\n",
      "rating                       0.60                0.62    1.00\n",
      "\n",
      "column-single\n",
      "                    selectedCount  selectedWithPaired  rating\n",
      "selectedCount                1.00                0.94    0.61\n",
      "selectedWithPaired           0.94                1.00    0.64\n",
      "rating                       0.61                0.64    1.00\n",
      "\n",
      "row-single\n",
      "                    selectedCount  selectedWithPaired  rating\n",
      "selectedCount                1.00                0.88    0.64\n",
      "selectedWithPaired           0.88                1.00    0.63\n",
      "rating                       0.64                0.63    1.00\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in gbr.result_layout.unique():   \n",
    "    print(i)\n",
    "    print(gbr.loc[gbr.result_layout==i,[\"selectedCount\",\"selectedWithPaired\",\"rating\"]].corr(\"pearson\").round(2))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "b3574ff0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>selectedCount</th>\n",
       "      <th>selectedWithPaired</th>\n",
       "      <th>userNormSelected</th>\n",
       "      <th>userNormSelectedWithPaired</th>\n",
       "      <th>rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>selectedCount</th>\n",
       "      <td>1.00</td>\n",
       "      <td>0.91</td>\n",
       "      <td>0.82</td>\n",
       "      <td>0.72</td>\n",
       "      <td>0.62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>selectedWithPaired</th>\n",
       "      <td>0.91</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.71</td>\n",
       "      <td>0.81</td>\n",
       "      <td>0.63</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>userNormSelected</th>\n",
       "      <td>0.82</td>\n",
       "      <td>0.71</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.88</td>\n",
       "      <td>0.58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>userNormSelectedWithPaired</th>\n",
       "      <td>0.72</td>\n",
       "      <td>0.81</td>\n",
       "      <td>0.88</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rating</th>\n",
       "      <td>0.62</td>\n",
       "      <td>0.63</td>\n",
       "      <td>0.58</td>\n",
       "      <td>0.60</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            selectedCount  selectedWithPaired  \\\n",
       "selectedCount                        1.00                0.91   \n",
       "selectedWithPaired                   0.91                1.00   \n",
       "userNormSelected                     0.82                0.71   \n",
       "userNormSelectedWithPaired           0.72                0.81   \n",
       "rating                               0.62                0.63   \n",
       "\n",
       "                            userNormSelected  userNormSelectedWithPaired  \\\n",
       "selectedCount                           0.82                        0.72   \n",
       "selectedWithPaired                      0.71                        0.81   \n",
       "userNormSelected                        1.00                        0.88   \n",
       "userNormSelectedWithPaired              0.88                        1.00   \n",
       "rating                                  0.58                        0.60   \n",
       "\n",
       "                            rating  \n",
       "selectedCount                 0.62  \n",
       "selectedWithPaired            0.63  \n",
       "userNormSelected              0.58  \n",
       "userNormSelectedWithPaired    0.60  \n",
       "rating                        1.00  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbr.corr().round(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf8519e3",
   "metadata": {},
   "source": [
    "- there is quite strong relation between low-level feedback and high-level algorithm rating \n",
    "- but still, it is far from identity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "847e4b91",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQcAAACGCAYAAADQME9uAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAASn0lEQVR4nO3df3RcdZnH8fenhWgzCSAWKwJtEVl2Od0DugHq4pmtYLFJoHURBXYpVLAVj7J4aPEItqK0qLu0rCygSKFGui6LCEq3SVtqAcEf0KYVJIis/OqxlZ/CwWTCEkuf/ePeKZPhZnKbzJ17kz6vc+bM3O+9M/dp0z65873P9/uVmeGcc+XGpB2Acy6bPDk45yJ5cnDORfLk4JyL5MnBORfJk4NzLtJeaQdQLePHj7fJkyenHYZzI87mzZtfMrMDytsTSw6SVgAnAy+Y2ZSwbX/gVmAy8AzwSTN7JeK95wALw80lZvb9wc43efJkOjs7qxO8c3sQSVuj2pP8WtEGzChr+xKwwcwOBzaE2/2ECeQy4DjgWOAySe9IME7nXITEkoOZ3Qe8XNY8CyheBXwf+FjEWz8KrDezl8OrivW8Ncm4EWTVqlWccsoprFq1Ku1QBjR//nzy+Tzz589PO5TMqHWfwwQzezZ8/RwwIeKYg4A/lGxvC9veQtI8YB7AxIkTqximq6bly5fz6quvsnz5cmbOnJl2OJE2bdrU7zlrWltb6e7u3rXd2NhIe3t7oudM7W6FBYM6hjWww8xuMLMmM2s64IC39Ke4jJg7dy777rsvc+fOTTuUAR1zzDH9nrOmu7ubVbev2fUoTRRJqfWVw/OSDjSzZyUdCLwQccx2YFrJ9sHAvTWIzSVk5syZmb1iKFq2bFnaIWROra8cVgHnhK/PAe6MOGYdcJKkd4QdkSeFbc65GkosOUi6BfgVcISkbZLOA74JTJf0e+Aj4TaSmiTdCGBmLwOLgU3h4/KwzTlXQ4l9rTCzMwfYdWLEsZ3Ap0u2VwArEgrNOReDl087R3A3oLW1Ne0wMmXUlE87Nxy16P0fafzKwTkXyZODS1xXVxcLFiygq6sr7VDcbvCvFS5xbW1tbNy4EYClS5emHE20sWPG8MbOnbS2tNDe0ZFaHC3NLfQUeiL3zfx4c7/tfD7/lmMacg10rKlO/J4cXOLmzJnT7zmL3ti5kzuWLOHUhQsHPzhBPYUeLp595ZDff+XKi6sWy6DJQdLxwENmVpB0FvAB4Gozixzm6Vy5KVOmZPaKwQ0sTp/Dd4BeSUcB84EngZsTjco5l7o4yWFHOEhqFnCtmV0HNCYblnMubXH6HLolXQKcBeQljQH2TjYs51za4lw5nA68DpxnZs8RjJIceo+Jc25EGPTKIUwIV0naJ5zCrQdYnXhkzrlUDXrlIOkzkp4DfgNsDh8+k2sGeHGRS1KcrxULgClmNtnMDg0f7006MDe4YnFRW1tb2qFUlPVBTa3NQXFRscahuL2ni9Mh+STQW60TSjqCYHr6ovcCXzGzb5UcM41gIpinw6Y7zOzyasUwWoyE4iLI/qCm7kKBleftmjGA2TfdmGI02REnOVwC/FLSgwQdkwCY2b8M5YRm9jhwNICksQTTwv044tD7zezkoZxjT+HFRS5JcZLDd4G7gUeAnVU+/4nAk15t6Vz2xEkOe5vZRQmd/wzglgH2fVDSw8AfgQVm9mhCMTjnIsTpkFwjaZ6kAyXtX3wM98SS6oCZwG0Ru7cAk8zsKOAa4CcDfMY8SZ2SOl988cXhhuScKxEnOZxJ2O9AdW9lNgNbzOz58h1m9mcz6wlfdwB7SxofcZyvW+FcQuIUQR2a0LnPZICvFJLeDTxvZibpWIIk9qeE4nDORYgzZLseuAiYaGbzJB0OHGFmQ66SlJQDpgOfKWk7H8DMrgdOAz4raQfwGnBGOPjLOVcjGuz/nKRbCb5KnG1mU8Jk8UszO7oG8cXW1NRknZ1euJklzS3NFHoK/dpyDTnWdKxJKaL+Wpqb6SkUBtzfkMvRsaa6sTY3t1AYYKanWsnlGlhTMluUpM1m1lR+XJy7FYeZ2emSzgQws15Jql6obrQq9BQ4d9m5/dpWzM/OciQ9hQLXn/CWZVR2Of/uDVU/Z6HQwyePqd5sTUPxw03xxk3G6ZDskzSOcNFbSYdRUgzlnBud4lw5fBVYCxwi6QfA8cCnkgzKOZe+OHcr7pK0GZgKCLjQzF5KPDLnXKriDNneYGZ/MrN2M1ttZi9Jqv6XMedcpgx45SDp7UA9MF7SOwiuGgD2AQ6qQWzOuRRV+lrxGeALwHsIypmL/gxcm2BMzrkMGDA5mNnVwNWSLjCza2oYk3MuAwbsc5B0Qvhyu6RTyx81is9VsHjxYvL5PIsXL047FDcKDVghKelrZnaZpO9F7DYzOzeiPTV7YoVk6VqJ9913X4qR9BdVGVmu2pWSLTNm0NNbtQnLhqShvp6OtWsrHjN16lSOfPfxNYoo2m+f+wUPPPDAru3drpA0s8vCZ69pyKjp06ezfv16pk+fnnYo/RR6Cky7dFrFY+79+r1VPWdPby+LSbdwd1GM5FRXV8cHDpuWfDAVPPHypljH+UK6I9iiRYtYtGhR2mG4USpO+bRzbg/kycE5F6lSEVTFOxJmdkf1w3HOZUWlPodTwud3AX9PMAM1wIcJpowbcnKQ9AzQDbxBsIp3U9l+AVcDLQRrZswxsy3ln+OcS06luxWfApB0F3CkmT0bbh8ItFXh3B+uMICrGTg8fBwHfCd8ds7VSJw+h0OKiSH0PDAxoXiKZgE3W+ABYL8wKTnnaiTOrcwNktbx5mSwpwM/HeZ5DbhLkgHfNbMbyvYfBPyhZHtb2FaapJA0D5gHMHFi0vkqW1pbWujueXO6scaGBto7Oiq8IzkzmmfQW+h/jz9OHUNpEVd9rp61ayoXELnaijOfw+cl/SNQ/EneYGZRy9ftjg+Z2XZJ7wLWS/qdme12iV+YVG6AoEJymDGNKN09Pay57ou7tps/92+pxdJb6GXS+ZOG9Rlbr/dFz7ImbhHUFqDbzH4qqV5So5kNeXVUM9sePr8g6cfAsUBpctgOHFKyfXDY5pyrkTiTvcwFfkSwZiYEl/c/GeoJJeUkNRZfAycBXWWHrQLOVmAq8GpZv4dzLmFxrhw+R/Cb/UEAM/t9+HVgqCYAPw4nsN4L+C8zW1u2bkUHwW3MJwhuZfr4DudqLE5yeN3M+oqz0Uvai3Am6qEws6eAoyLary95bQRJyTmXkji3Mn8m6VJgnKTpBAvf/k+yYTnn0hYnOXwJeBF4hGDquA4z+3KiUTnnUhfna8UF4ZRxy4sNki4M25xzo1ScK4dzItrmVDmOzOnq6mLBggV0dZXfSHFuz1BpVOaZwD8Bh0paVbKrEXg56cDS1tbWxsaNGwFYunRpytG8qbWlme5wCrbywqd8Pk9jQ472Ki9UG1UBWa4aRUylFZPlBqug7Ovr466/7Bh2DMPRt/fgF+K5XEPstSqTkss1xDqu0p/mlwTlyuOBZSXt3cBvhhzZCDFnzpx+z1nR3VPgzkubB9w/6+vVX8G6t9DLG594o+qfu1sx3FY5OdXV1XFS3dtqFE20+2PcxCtd3TpKPp/n4tlDTx5Xrry4avOJVhqVuRXYKumfgT+a2f8BhIvqHgw8U5UIMmrKlCmZumJwrtbi9Dn8ENhZsv0Gwe1M59woFic57GVmfcWN8HVdciE557IgTnJ4UdLM4oakWYCvsu3cKBenzuF84AeSriMom94GnJ1oVM651MWZz+FJYKqkhnC7Z5C3OOdGgThDtidIugm4zcx6JB0p6bwaxJaq2bNnk8/nmT17dtqhOJeKOF8r2oDvAcXxFP8L3ArclFBMmbB169Z+z0lpbf4o3YXXdus9g9UyVComitKYG0f7mnUD7u/r6+P1X7++W59ZbW/rq1zD0FBfH2s5uiQ11NcP/zNyDVy58uJhvb9a4iSH8Wb2Q0mXAJjZDklDroiRdAhwM8G8DkYw7dzVZcdMA+4Eng6b7jCzy4d6zqGYNGkSW7duZdKk4U1/Npjuwmv850deSfQcgzlrkBlB6+rqGPv+sbUJZgBjn6h8/sEWsIUgaV516HuHHMNFTz+V+ILFHWVFUvl8nmuu+P6Ax1/w5XMSiylOcihIeifhHA7FmZmGcc4dwHwz2xLOCLVZ0noz+23Zcfeb2cnDOM+wrFy5Mq1TO5cJcZLDRQTTth0m6RfAAcBpQz1hON3bs+HrbkmPEUw9V54cnHMpinO3YoukfwCOAAQ8bmZ/qcbJJU0G3k84BV2ZD0p6GPgjsMDMHq3GOeNobW2lu/vN+XMbGxtpb2+v1emdy4ShrJX5V5KGvVZmeGv0duALZvbnst1bgEnh3ZEWggltD4/4jETWreju7mbl7at3bc/+eGrfbpxLTZy1MqMYw1src2+CxPCDqCRTmizMrEPStyWNL18+b09et8K5pA26Vma1hYvk3gQ8ZmZXDXDMu4HnzcwkHUtQj/GnJOJxzkUbtM9B0gTg68B7zKxZ0pHAB81sqHUOxwOzgUckPRS2XUq4/mY4C/VpwGcl7QBeA84IZ6R2ztVIzYugzOznBB2blY65Frh2KJ+/O2Y0t9BbiK4GL+9niCosqs81sHaQyTsG09fXx22PpTuRSl9fX8X99bn6QSdbSVp9bvgFRm731LwIKkt6Cz0cdfZlQ37/wzd/bdgx1NXV8Ym/SbfA6M7tlUfgD7bAbT6fr8pamUkXGLndE2fIdrWLoJxzI0DNi6CccyNDqkVQzrnsGvBrhaRjwluKmNkO4O+AK4BlkvavUXzOuZRU6nP4LtAHICkPfJNgNOWrhIVHzrnRq9LXirFmVly85nSCodW3A7eX1Cc450apSlcOYyUVk8eJwN0l++J0ZDrnRrBK/8lvAX4m6SWCKsX7ASS9D7+V6dyoV2lsxRWSNgAHAneVlC+PAS6oRXDD8dHmZl4rFAY9briFTJWmZBuXy7FuTeUp3Rpz4wadiSlpjblxw3p/fa5+2Gtl1qICsqG+nouefmpY76+1hoYGLvhy1FrWb+5PikbLkIWmpibr7OzctZ3P5ykc9+kUI4LcgzdWveovn88PulZmVioN8/k85y47t1/bivkrMhNfqXw+zx1LlnDqwoWZjC9JkjabWVN5e5wKSefcHsiTg3MukicH51wkTw7OuUipJAdJMyQ9LukJSV+K2P82SbeG+x8MJ6J1ztVQzZODpLHAdUAzcCRwZji7VKnzgFfM7H3AvwP/WtsonXNpXDkcCzxhZk+ZWR/w38CssmNmAcVlfn4EnBjOPencHqvW67emUQZ9EPCHku1twHEDHRPOPPUq8E6g3+zTlaam19ixcHe/VfZqTvvsU/XPbGzIVVwrs7EhV/VzDlWuIceK+Sve0pZFY8eM4dSFC2lMsKhouGq1fmvRiB4jUWlq+p/dc08qMSWtvaNyxWWWrBlBsd5z771phzCoWq3fWpRGctgOHFKyfXDYFnXMtnDw17741PRuD1fr9VvT6HPYBBwu6VBJdcAZBNPQlVoFFAvKTwPu9qnpnautml85hH0InwfWAWOBFWb2qKTLgU4zW0Uw7f1KSU8ALxMkEOdcDY2agVeSXgSq3VMznrJO0AwaCTHCyIhzT41xkpkdUN44apJDEiR1Ro1Wy5KRECOMjDg9xv68fNo5F8mTg3MukieHykbCLNsjIUYYGXF6jCW8z8E5F8mvHJxzkTw5hCStkPSCpK6y9jmSJmdh4JekQyTdI+m3kh6VdGHJvizF+XZJGyU9HMb5tWJckr4aPqceJwSjhCX9WtLqLMYo6RlJj0h6SFJnSXviP29PDm9qA2YUNyQdJOlGgjLuDwHXpxRXqR3AfDM7EpgKfE7S8RmM83XgBDM7Cjia4O91qqT/APaX9DGCpRWz4ELgsfD10RmN8cNmdrSZNdX036WZ+SN8AJOBrpLtCcAzwGpgTNrxRcR7JzA9y3EC9cAWgpG3fw28Anwn7bjC2A4GNgAnAKvDtqzF+AwwvqytJj9vv3IYgKT3AEuAFcCtBBPUZEY4O9b7gcfJYJzh5fpDwAvAeoKric8CK4F1kpakGF7Rt4AvAjsBJB1N9mI04C5JmyXNq+m/y7QzY5YelF05hG1zwnalHV9JTA3AZuDULMcZxrUfcA/wt+H2V8PnVOMETga+Hb6eRvBbuHj3LhMxhjEcFD6/C3gYyNfq5+23MkuEv41Xm9mUtGMZiKS9Cf4hrzOzq9KOJw5JXwF6zWxp2rEUSfoGMJugH+ftwD7AHWZ2VqqBVRB2lPbU6u/Rv1aMIGHP9E3AY1lODJIOkLRf+HocQb/I71INqoyZXWJmB5vZZIJRv3dnLTFIyklqLL4GTgK6Kr+rejw5hCTdAvwKOELSNknnpR1ThOMJftudEN7aekhSS9pBRTgQuEfSbwjm71hvZqtTjmkkmgD8XNLDwEag3czW1urk/rXCORfJrxycc5E8OTjnInlycM5F8uTgnIvkycE5F8mTg3MukicH51wkTw7OuUj/D/EjqdMWbvnnAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 266.4x136.8 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "f = plt.figure()\n",
    "f.set_figwidth(3.7)\n",
    "f.set_figheight(1.9)\n",
    "\n",
    "sns.boxenplot(data = [gbr.loc[gbr.rating == 1, \"selectedCount\"],\n",
    "              gbr.loc[gbr.rating == 2, \"selectedCount\"],\n",
    "              gbr.loc[gbr.rating == 3, \"selectedCount\"],\n",
    "              gbr.loc[gbr.rating == 4, \"selectedCount\"],\n",
    "              gbr.loc[gbr.rating == 5, \"selectedCount\"]], linewidth=1, flier_kws={\"marker\":\"o\", \"s\":2})\n",
    "plt.ylim([-0.5, 10.5])\n",
    "ax = plt.gca()\n",
    "ax.set_xticklabels([\"1*\",\"2*\",\"3*\",\"4*\",\"5*\"])\n",
    "plt.ylabel(\"Selected items\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "e6743b3d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1063, 1)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rcd = relative_comparison_data.set_index([\"participation\",\"iteration\"])[[\"relative_comparison_numeric\"]]\n",
    "rcd.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "2e0ae498",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>variant</th>\n",
       "      <th>selectedCount</th>\n",
       "      <th>selectedWithPaired</th>\n",
       "      <th>userNormSelected</th>\n",
       "      <th>userNormSelectedWithPaired</th>\n",
       "      <th>rating</th>\n",
       "      <th>result_layout</th>\n",
       "      <th>layout_high_level</th>\n",
       "      <th>variant_v1</th>\n",
       "      <th>selectedCount_v1</th>\n",
       "      <th>selectedWithPaired_v1</th>\n",
       "      <th>userNormSelected_v1</th>\n",
       "      <th>userNormSelectedWithPaired_v1</th>\n",
       "      <th>rating_v1</th>\n",
       "      <th>result_layout_v1</th>\n",
       "      <th>layout_high_level_v1</th>\n",
       "      <th>difference</th>\n",
       "      <th>differenceWithPaired</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>participation</th>\n",
       "      <th>iteration</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">36</th>\n",
       "      <th>1.0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.1875</td>\n",
       "      <td>1.3125</td>\n",
       "      <td>4.0</td>\n",
       "      <td>row-single-scrollable</td>\n",
       "      <td>rows</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.1875</td>\n",
       "      <td>-0.6875</td>\n",
       "      <td>4.0</td>\n",
       "      <td>row-single-scrollable</td>\n",
       "      <td>rows</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.1875</td>\n",
       "      <td>2.3125</td>\n",
       "      <td>4.0</td>\n",
       "      <td>columns</td>\n",
       "      <td>cols</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>-1.8125</td>\n",
       "      <td>2.3125</td>\n",
       "      <td>4.0</td>\n",
       "      <td>columns</td>\n",
       "      <td>cols</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.8125</td>\n",
       "      <td>-1.6875</td>\n",
       "      <td>2.0</td>\n",
       "      <td>max-columns</td>\n",
       "      <td>cols</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>-0.8125</td>\n",
       "      <td>-0.6875</td>\n",
       "      <td>3.0</td>\n",
       "      <td>max-columns</td>\n",
       "      <td>cols</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.1875</td>\n",
       "      <td>1.3125</td>\n",
       "      <td>3.0</td>\n",
       "      <td>rows</td>\n",
       "      <td>rows</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>-1.8125</td>\n",
       "      <td>-0.6875</td>\n",
       "      <td>2.0</td>\n",
       "      <td>rows</td>\n",
       "      <td>rows</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5.0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.8125</td>\n",
       "      <td>-1.6875</td>\n",
       "      <td>2.0</td>\n",
       "      <td>row-single-scrollable</td>\n",
       "      <td>rows</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.8125</td>\n",
       "      <td>-1.6875</td>\n",
       "      <td>2.0</td>\n",
       "      <td>row-single-scrollable</td>\n",
       "      <td>rows</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         variant  selectedCount  selectedWithPaired  \\\n",
       "participation iteration                                               \n",
       "36            1.0            0.0            4.0                 4.0   \n",
       "              2.0            0.0            5.0                 5.0   \n",
       "              3.0            0.0            1.0                 1.0   \n",
       "              4.0            0.0            4.0                 4.0   \n",
       "              5.0            0.0            1.0                 1.0   \n",
       "\n",
       "                         userNormSelected  userNormSelectedWithPaired  rating  \\\n",
       "participation iteration                                                         \n",
       "36            1.0                  2.1875                      1.3125     4.0   \n",
       "              2.0                  3.1875                      2.3125     4.0   \n",
       "              3.0                 -0.8125                     -1.6875     2.0   \n",
       "              4.0                  2.1875                      1.3125     3.0   \n",
       "              5.0                 -0.8125                     -1.6875     2.0   \n",
       "\n",
       "                                 result_layout layout_high_level  variant_v1  \\\n",
       "participation iteration                                                        \n",
       "36            1.0        row-single-scrollable              rows         1.0   \n",
       "              2.0                      columns              cols         1.0   \n",
       "              3.0                  max-columns              cols         1.0   \n",
       "              4.0                         rows              rows         1.0   \n",
       "              5.0        row-single-scrollable              rows         1.0   \n",
       "\n",
       "                         selectedCount_v1  selectedWithPaired_v1  \\\n",
       "participation iteration                                            \n",
       "36            1.0                     2.0                    2.0   \n",
       "              2.0                     0.0                    5.0   \n",
       "              3.0                     1.0                    2.0   \n",
       "              4.0                     0.0                    2.0   \n",
       "              5.0                     1.0                    1.0   \n",
       "\n",
       "                         userNormSelected_v1  userNormSelectedWithPaired_v1  \\\n",
       "participation iteration                                                       \n",
       "36            1.0                     0.1875                        -0.6875   \n",
       "              2.0                    -1.8125                         2.3125   \n",
       "              3.0                    -0.8125                        -0.6875   \n",
       "              4.0                    -1.8125                        -0.6875   \n",
       "              5.0                    -0.8125                        -1.6875   \n",
       "\n",
       "                         rating_v1       result_layout_v1  \\\n",
       "participation iteration                                     \n",
       "36            1.0              4.0  row-single-scrollable   \n",
       "              2.0              4.0                columns   \n",
       "              3.0              3.0            max-columns   \n",
       "              4.0              2.0                   rows   \n",
       "              5.0              2.0  row-single-scrollable   \n",
       "\n",
       "                        layout_high_level_v1  difference  differenceWithPaired  \n",
       "participation iteration                                                         \n",
       "36            1.0                       rows         2.0                   2.0  \n",
       "              2.0                       cols         5.0                   0.0  \n",
       "              3.0                       cols         0.0                  -1.0  \n",
       "              4.0                       rows         4.0                   2.0  \n",
       "              5.0                       rows         0.0                   0.0  "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "countSelections = gbr.reset_index(2)\n",
    "v0 = countSelections.loc[countSelections.variant==0]\n",
    "v1 = countSelections.loc[countSelections.variant==1]\n",
    "variant_joined = v0.join(v1, how=\"outer\", rsuffix=\"_v1\")\n",
    "variant_joined.fillna(0, inplace=True)\n",
    "variant_joined[\"difference\"] = variant_joined[\"selectedCount\"] - variant_joined[\"selectedCount_v1\"] \n",
    "variant_joined[\"differenceWithPaired\"] = variant_joined[\"selectedWithPaired\"] - variant_joined[\"selectedWithPaired_v1\"] \n",
    "variant_joined.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "ab2f55c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1063, 19)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rcd = rcd.join(variant_joined, how=\"inner\")\n",
    "rcd.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "8836a5fb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>relative_comparison_numeric</th>\n",
       "      <th>variant</th>\n",
       "      <th>selectedCount</th>\n",
       "      <th>selectedWithPaired</th>\n",
       "      <th>userNormSelected</th>\n",
       "      <th>userNormSelectedWithPaired</th>\n",
       "      <th>rating</th>\n",
       "      <th>result_layout</th>\n",
       "      <th>layout_high_level</th>\n",
       "      <th>variant_v1</th>\n",
       "      <th>selectedCount_v1</th>\n",
       "      <th>selectedWithPaired_v1</th>\n",
       "      <th>userNormSelected_v1</th>\n",
       "      <th>userNormSelectedWithPaired_v1</th>\n",
       "      <th>rating_v1</th>\n",
       "      <th>result_layout_v1</th>\n",
       "      <th>layout_high_level_v1</th>\n",
       "      <th>difference</th>\n",
       "      <th>differenceWithPaired</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>participation</th>\n",
       "      <th>iteration</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">36</th>\n",
       "      <th>1.0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.1875</td>\n",
       "      <td>1.3125</td>\n",
       "      <td>4.0</td>\n",
       "      <td>row-single-scrollable</td>\n",
       "      <td>rows</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.1875</td>\n",
       "      <td>-0.6875</td>\n",
       "      <td>4.0</td>\n",
       "      <td>row-single-scrollable</td>\n",
       "      <td>rows</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.1875</td>\n",
       "      <td>2.3125</td>\n",
       "      <td>4.0</td>\n",
       "      <td>columns</td>\n",
       "      <td>cols</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>-1.8125</td>\n",
       "      <td>2.3125</td>\n",
       "      <td>4.0</td>\n",
       "      <td>columns</td>\n",
       "      <td>cols</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.8125</td>\n",
       "      <td>-1.6875</td>\n",
       "      <td>2.0</td>\n",
       "      <td>max-columns</td>\n",
       "      <td>cols</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>-0.8125</td>\n",
       "      <td>-0.6875</td>\n",
       "      <td>3.0</td>\n",
       "      <td>max-columns</td>\n",
       "      <td>cols</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>-1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.1875</td>\n",
       "      <td>1.3125</td>\n",
       "      <td>3.0</td>\n",
       "      <td>rows</td>\n",
       "      <td>rows</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>-1.8125</td>\n",
       "      <td>-0.6875</td>\n",
       "      <td>2.0</td>\n",
       "      <td>rows</td>\n",
       "      <td>rows</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5.0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.8125</td>\n",
       "      <td>-1.6875</td>\n",
       "      <td>2.0</td>\n",
       "      <td>row-single-scrollable</td>\n",
       "      <td>rows</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.8125</td>\n",
       "      <td>-1.6875</td>\n",
       "      <td>2.0</td>\n",
       "      <td>row-single-scrollable</td>\n",
       "      <td>rows</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">261</th>\n",
       "      <th>4.0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.7500</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>3.0</td>\n",
       "      <td>max-columns</td>\n",
       "      <td>cols</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.7500</td>\n",
       "      <td>0.5000</td>\n",
       "      <td>3.0</td>\n",
       "      <td>max-columns</td>\n",
       "      <td>cols</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5.0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.2500</td>\n",
       "      <td>-1.5000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>rows</td>\n",
       "      <td>rows</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>-0.2500</td>\n",
       "      <td>-0.5000</td>\n",
       "      <td>3.0</td>\n",
       "      <td>rows</td>\n",
       "      <td>rows</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6.0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.7500</td>\n",
       "      <td>1.5000</td>\n",
       "      <td>3.0</td>\n",
       "      <td>row-single-scrollable</td>\n",
       "      <td>rows</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>-0.2500</td>\n",
       "      <td>1.5000</td>\n",
       "      <td>4.0</td>\n",
       "      <td>row-single-scrollable</td>\n",
       "      <td>rows</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7.0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>-0.2500</td>\n",
       "      <td>-0.5000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>columns</td>\n",
       "      <td>cols</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.7500</td>\n",
       "      <td>1.5000</td>\n",
       "      <td>3.0</td>\n",
       "      <td>columns</td>\n",
       "      <td>cols</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8.0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>-0.2500</td>\n",
       "      <td>-0.5000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>max-columns</td>\n",
       "      <td>cols</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-1.2500</td>\n",
       "      <td>-1.5000</td>\n",
       "      <td>1.0</td>\n",
       "      <td>max-columns</td>\n",
       "      <td>cols</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1063 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                         relative_comparison_numeric  variant  selectedCount  \\\n",
       "participation iteration                                                        \n",
       "36            1.0                                  0      0.0            4.0   \n",
       "              2.0                                  0      0.0            5.0   \n",
       "              3.0                                  1      0.0            1.0   \n",
       "              4.0                                 -1      0.0            4.0   \n",
       "              5.0                                  0      0.0            1.0   \n",
       "...                                              ...      ...            ...   \n",
       "261           4.0                                  0      0.0            3.0   \n",
       "              5.0                                  1      0.0            1.0   \n",
       "              6.0                                  1      0.0            4.0   \n",
       "              7.0                                  1      0.0            2.0   \n",
       "              8.0                                  0      0.0            2.0   \n",
       "\n",
       "                         selectedWithPaired  userNormSelected  \\\n",
       "participation iteration                                         \n",
       "36            1.0                       4.0            2.1875   \n",
       "              2.0                       5.0            3.1875   \n",
       "              3.0                       1.0           -0.8125   \n",
       "              4.0                       4.0            2.1875   \n",
       "              5.0                       1.0           -0.8125   \n",
       "...                                     ...               ...   \n",
       "261           4.0                       3.0            0.7500   \n",
       "              5.0                       1.0           -1.2500   \n",
       "              6.0                       4.0            1.7500   \n",
       "              7.0                       2.0           -0.2500   \n",
       "              8.0                       2.0           -0.2500   \n",
       "\n",
       "                         userNormSelectedWithPaired  rating  \\\n",
       "participation iteration                                       \n",
       "36            1.0                            1.3125     4.0   \n",
       "              2.0                            2.3125     4.0   \n",
       "              3.0                           -1.6875     2.0   \n",
       "              4.0                            1.3125     3.0   \n",
       "              5.0                           -1.6875     2.0   \n",
       "...                                             ...     ...   \n",
       "261           4.0                            0.5000     3.0   \n",
       "              5.0                           -1.5000     1.0   \n",
       "              6.0                            1.5000     3.0   \n",
       "              7.0                           -0.5000     1.0   \n",
       "              8.0                           -0.5000     1.0   \n",
       "\n",
       "                                 result_layout layout_high_level  variant_v1  \\\n",
       "participation iteration                                                        \n",
       "36            1.0        row-single-scrollable              rows         1.0   \n",
       "              2.0                      columns              cols         1.0   \n",
       "              3.0                  max-columns              cols         1.0   \n",
       "              4.0                         rows              rows         1.0   \n",
       "              5.0        row-single-scrollable              rows         1.0   \n",
       "...                                        ...               ...         ...   \n",
       "261           4.0                  max-columns              cols         1.0   \n",
       "              5.0                         rows              rows         1.0   \n",
       "              6.0        row-single-scrollable              rows         1.0   \n",
       "              7.0                      columns              cols         1.0   \n",
       "              8.0                  max-columns              cols         1.0   \n",
       "\n",
       "                         selectedCount_v1  selectedWithPaired_v1  \\\n",
       "participation iteration                                            \n",
       "36            1.0                     2.0                    2.0   \n",
       "              2.0                     0.0                    5.0   \n",
       "              3.0                     1.0                    2.0   \n",
       "              4.0                     0.0                    2.0   \n",
       "              5.0                     1.0                    1.0   \n",
       "...                                   ...                    ...   \n",
       "261           4.0                     3.0                    3.0   \n",
       "              5.0                     2.0                    2.0   \n",
       "              6.0                     2.0                    4.0   \n",
       "              7.0                     3.0                    4.0   \n",
       "              8.0                     1.0                    1.0   \n",
       "\n",
       "                         userNormSelected_v1  userNormSelectedWithPaired_v1  \\\n",
       "participation iteration                                                       \n",
       "36            1.0                     0.1875                        -0.6875   \n",
       "              2.0                    -1.8125                         2.3125   \n",
       "              3.0                    -0.8125                        -0.6875   \n",
       "              4.0                    -1.8125                        -0.6875   \n",
       "              5.0                    -0.8125                        -1.6875   \n",
       "...                                      ...                            ...   \n",
       "261           4.0                     0.7500                         0.5000   \n",
       "              5.0                    -0.2500                        -0.5000   \n",
       "              6.0                    -0.2500                         1.5000   \n",
       "              7.0                     0.7500                         1.5000   \n",
       "              8.0                    -1.2500                        -1.5000   \n",
       "\n",
       "                         rating_v1       result_layout_v1  \\\n",
       "participation iteration                                     \n",
       "36            1.0              4.0  row-single-scrollable   \n",
       "              2.0              4.0                columns   \n",
       "              3.0              3.0            max-columns   \n",
       "              4.0              2.0                   rows   \n",
       "              5.0              2.0  row-single-scrollable   \n",
       "...                            ...                    ...   \n",
       "261           4.0              3.0            max-columns   \n",
       "              5.0              3.0                   rows   \n",
       "              6.0              4.0  row-single-scrollable   \n",
       "              7.0              3.0                columns   \n",
       "              8.0              1.0            max-columns   \n",
       "\n",
       "                        layout_high_level_v1  difference  differenceWithPaired  \n",
       "participation iteration                                                         \n",
       "36            1.0                       rows         2.0                   2.0  \n",
       "              2.0                       cols         5.0                   0.0  \n",
       "              3.0                       cols         0.0                  -1.0  \n",
       "              4.0                       rows         4.0                   2.0  \n",
       "              5.0                       rows         0.0                   0.0  \n",
       "...                                      ...         ...                   ...  \n",
       "261           4.0                       cols         0.0                   0.0  \n",
       "              5.0                       rows        -1.0                  -1.0  \n",
       "              6.0                       rows         2.0                   0.0  \n",
       "              7.0                       cols        -1.0                  -2.0  \n",
       "              8.0                       cols         1.0                   1.0  \n",
       "\n",
       "[1063 rows x 19 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rcd = rcd.loc[rcd.difference >=-10].loc[rcd.difference <=10]\n",
    "rcd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "1dbd3266",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(12, 19)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "leftFeedbackInconsistence = rcd.loc[((rcd.relative_comparison_numeric < 0)&(rcd.difference < 0))]\n",
    "leftFeedbackInconsistence.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "be49aed9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(42, 19)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rightFeedbackInconsistence = rcd.loc[((rcd.relative_comparison_numeric > 0)&(rcd.difference > 0))]\n",
    "rightFeedbackInconsistence.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "047f530e",
   "metadata": {},
   "outputs": [],
   "source": [
    "hardInconsistencies = leftFeedbackInconsistence.shape[0] + rightFeedbackInconsistence.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "2dfa8f28",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.05079962370649106"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hardInconsistencies / rcd.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5fff9e6",
   "metadata": {},
   "source": [
    "- neither type of inconsistencies is very much common (advantaged / disadvantaged algorithm is contradictory to what low-level feedback considers as better)\n",
    "- there seems to be some bias towards right inconsistency, which is mainly caused by the selections of mutually recommended items"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "d5734cd5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>relative_comparison_numeric</th>\n",
       "      <th>difference</th>\n",
       "      <th>differenceWithPaired</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>relative_comparison_numeric</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.723692</td>\n",
       "      <td>-0.773985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>difference</th>\n",
       "      <td>-0.723692</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.886161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>differenceWithPaired</th>\n",
       "      <td>-0.773985</td>\n",
       "      <td>0.886161</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             relative_comparison_numeric  difference  \\\n",
       "relative_comparison_numeric                     1.000000   -0.723692   \n",
       "difference                                     -0.723692    1.000000   \n",
       "differenceWithPaired                           -0.773985    0.886161   \n",
       "\n",
       "                             differenceWithPaired  \n",
       "relative_comparison_numeric             -0.773985  \n",
       "difference                               0.886161  \n",
       "differenceWithPaired                     1.000000  "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rcd[[\"relative_comparison_numeric\",\"difference\",\"differenceWithPaired\"]].corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "59fd8cdc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "row-single-scrollable\n",
      "                             difference  differenceWithPaired  \\\n",
      "difference                         1.00                  0.89   \n",
      "differenceWithPaired               0.89                  1.00   \n",
      "relative_comparison_numeric       -0.72                 -0.78   \n",
      "\n",
      "                             relative_comparison_numeric  \n",
      "difference                                         -0.72  \n",
      "differenceWithPaired                               -0.78  \n",
      "relative_comparison_numeric                         1.00  \n",
      "\n",
      "columns\n",
      "                             difference  differenceWithPaired  \\\n",
      "difference                         1.00                  0.93   \n",
      "differenceWithPaired               0.93                  1.00   \n",
      "relative_comparison_numeric       -0.73                 -0.76   \n",
      "\n",
      "                             relative_comparison_numeric  \n",
      "difference                                         -0.73  \n",
      "differenceWithPaired                               -0.76  \n",
      "relative_comparison_numeric                         1.00  \n",
      "\n",
      "max-columns\n",
      "                             difference  differenceWithPaired  \\\n",
      "difference                         1.00                  0.94   \n",
      "differenceWithPaired               0.94                  1.00   \n",
      "relative_comparison_numeric       -0.74                 -0.79   \n",
      "\n",
      "                             relative_comparison_numeric  \n",
      "difference                                         -0.74  \n",
      "differenceWithPaired                               -0.79  \n",
      "relative_comparison_numeric                         1.00  \n",
      "\n",
      "rows\n",
      "                             difference  differenceWithPaired  \\\n",
      "difference                         1.00                  0.83   \n",
      "differenceWithPaired               0.83                  1.00   \n",
      "relative_comparison_numeric       -0.73                 -0.81   \n",
      "\n",
      "                             relative_comparison_numeric  \n",
      "difference                                         -0.73  \n",
      "differenceWithPaired                               -0.81  \n",
      "relative_comparison_numeric                         1.00  \n",
      "\n",
      "column-single\n",
      "                             difference  differenceWithPaired  \\\n",
      "difference                         1.00                  0.91   \n",
      "differenceWithPaired               0.91                  1.00   \n",
      "relative_comparison_numeric       -0.72                 -0.78   \n",
      "\n",
      "                             relative_comparison_numeric  \n",
      "difference                                         -0.72  \n",
      "differenceWithPaired                               -0.78  \n",
      "relative_comparison_numeric                         1.00  \n",
      "\n",
      "row-single\n",
      "                             difference  differenceWithPaired  \\\n",
      "difference                         1.00                  0.82   \n",
      "differenceWithPaired               0.82                  1.00   \n",
      "relative_comparison_numeric       -0.72                 -0.75   \n",
      "\n",
      "                             relative_comparison_numeric  \n",
      "difference                                         -0.72  \n",
      "differenceWithPaired                               -0.75  \n",
      "relative_comparison_numeric                         1.00  \n",
      "\n",
      "0\n",
      "                             difference  differenceWithPaired  \\\n",
      "difference                          1.0                   1.0   \n",
      "differenceWithPaired                1.0                   1.0   \n",
      "relative_comparison_numeric        -1.0                  -1.0   \n",
      "\n",
      "                             relative_comparison_numeric  \n",
      "difference                                          -1.0  \n",
      "differenceWithPaired                                -1.0  \n",
      "relative_comparison_numeric                          1.0  \n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in rcd.result_layout.unique():   \n",
    "    print(i)\n",
    "    print(rcd.loc[rcd.result_layout==i,[\"difference\",\"differenceWithPaired\",\"relative_comparison_numeric\"]].corr(\"pearson\").round(2))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "4bff380a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfEAAACXCAYAAAAF6GKOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAApTUlEQVR4nO3deZxcVZn/8c+Xhmh6AUVQAUmCiDiY34AaI4rTEvbumEZAjCjRCIaJiivRUUlcCDIoA26gkcbYEBltGQg0ZCNAAozIEpClAVEEogQUBAe7O0hDeH5/3NNJpaita7n3VvXzfr36VXXX83RV3Xvucu5zZGY455xzrv5sk3QAzjnnnCuPV+LOOedcnfJK3DnnnKtTXok755xzdcorceecc65OeSXunHPO1SmvxGtE0qmS7pV0t6Q7Jb2jyPw9kt5fRjmTJH2ojOVylifpNEmHhPefk9Q82nXXA0mvkPTJjOFdJf1PkjG53HxbSjdJB0q6qsg8+0nqzBjukvTl2keXDEkXSNonjrK8Eq8BSe8E3gu81cz+FTgE+HONipsEjHrHk4+Zfc3MrgmDnwPqdscjadsCk18BbK7EzewxMxv1jt/Vlm9L6VBkWyrFfsDmStzM+szszArXmUqSmszs42Z2XxzleSVeG7sAfzOz5wDM7G9m9hiApLdJul7S7ZJWSdole+F880h6g6RrJN0l6Q5JewJnAv8WzlA+L6lJ0lmSbgtnLv8elpWkcyU9IOka4NW5Ah85q5D0GWBXYI2kNWHaYZJ+E8q+RFJrGP+IpP8MMayT9NYQ9x8lzQ3z7CLphjBPv6R/yyr3CEmXZAwfKOmq8P/0hGXukfT5Qh+8pNmS+iRdB1wrqVXStSHmeyQdGWY9E9gzxHNWOAvrz1jHZZJWSvqDpO9krP9ESb+XdKukbknnForHVcy3pfRsSy2SFoff/m8ztqXMZaaG/+u3km6StLekccBpwMwQ88yw7nMl7SBpvaRtwvItkv4saTtJe4Zt8HZJN0p6U47yWiX9LPw/d0s6Jow/Lozrl/TtjPkHw3d6b/j+p0paK+khSV0Z//cVYfwfJH09Y/nLQzz3Sjopa71nS7oLeGdYdkq+z1zRlYmbQ8xLJb0yjF8r6dvhM/599nebk5n5X5X/gFbgTuD3wI+A94Tx2wE3ATuH4ZnA4vC+B3h/kXluAY4K719OdGR/IHBVRtknAfPD+5cB64A9gKOB1UAT0Q7l/4D354i9Z2Q88AiwU3i/E3AD0BKG/wP4WsZ8nwjvvwvcDbQBOwN/DeNPAU4N75uAtqxytwX+lLH+HwPHA28DVmfM94oin/1s4FFgx4z1bp/xPzwIiOisqz9juc3DYR0PATuEz3k9sHv43B4Bdgzf043AuUn/3hr5z7elVG1LZwDHjywbvpOWzM8N2B7YNrw/BLg0Y13nZq373PD+CmBaxnd0QXh/LbBXeP8O4LocMX4b+F7G8CvDd/Kn8JltC1wHvC9MN6AjvF8KXE30O9kXuDMjtseBVwHjgX5gSpg28lmMjH9Vxno/kBHHWmBKvs88fK8jv+XTRv6HsNzZ4X0ncE2xbaTSSyQuBzMblPQ24N+AaUCvovs/64DJwGpJEG2Aj2ctvneueSS1AbuZ2dJQxj8BwjyZDgP+VVvu0e0A7AW0A78ws03AY+HoejT2B/YBfh3KHAf8JmN6X3i9B2g1swFgQNJzkl4B3AYslrQdcLmZ3Zm5cjN7QdJKYIaie9PTgS8RbYSvl/RDYBnRRlfMajN7OrwXcIakduBFYDfgNSWs41ozewZA0n3ARKKd7/Uj6w5nO28sYV2uTL4tpWpbOgzokjQvDL8cmJA1/w7AhZL2IqrYtiuhjF6iynsN8EHgR4quTLwLuCTje3lZjmUPCcsAYGZ/D9v6WjN7EkDSxUTf2eXAMLAyzH4P8JyZPS/pHqID+cz/+6mw/GXAu4l+c5+RdFSYZ3ei38NTwCbg0hzxPUTWZy5pB6LK/Powz4XAJRnLXBZeb8+KKSevxGskbOBrgbXhB/JRoi/lXjN7Z4FFlWuesOMphYBPm9mqrOU788xfKhH9sI/LM/258PpixvuR4W3N7IawcU0HeiSdY2YXZa3jl8DJwNPAurDzQtK+wOHAXOADwAlFYh3KeP9hoiPyt4WN9RGinU8xmf/DJnxbSYxvS5slvS0JOMbMHtjqn5EyD4oXAmvM7ChJk4i+t2L6iA60dyQ6c72O6Az//8xsvxKWH43nLZzmkvH5mtmL2vq+f3anIibpQKKDhnea2UZJa9myL/ln+J1uvVB0UJH9mRe8jcGW77yk/Y7fE68BRfeB9soYtR/RJdkHgJ0VNdYh3Pd5c9biOecJG+Gjkt4Xxr9MUWvXAaLLbSNWAZ8IR+lIeqOkFqLLdzPDPZpdiM5qislc983AAZLeENbbIqnks1BJE4kuB3YDFwBvzTHb9WH8HKKdEJJ2ArYxs0uB+XmWK2QH4IlQgU8jOqPO/t9KdRvwHkmvDBv8MaNc3o2Sb0svleC2tAr4tMKpsaS35JhnB2BDeD87Y3ze7c3MBom2re8TXZbfZGb/AB6WdGwoS6EyzLYa+NTIQLi3fCvRdrqTpCbgOKLPYzQOlbSjpPHA+4Bfh//t76ECfxPRFZWCcn3m4Qrf37XlfvesMuLbzM8uaqMV+GG49PUC0X3Yk8xsOFya+0G4pLIt8D3g3pEFi8wzC/iJpNOA54Fjie6tbFLUoKKHaEOYBNwRNrYniX6ES4GDgPuI7hdlXr7L53xgpaTHzGyapNnALySNXNaaT3RfrBQHAl+U9DwwCHwkewYz26ToUZXZRGdbEF3+/plCwxfgKwAKjXzMbFGRci8GrgxncOuA34XlnpL0a0WN2VYA5xX7B8xsg6QziHYST4d1PVNsOVcR35Ze6kCS2ZYWEn1+d4d1PEz05ECm7xBdTp9PdPl4xBrgy5LuBP4zx7p7iS4pH5gx7sPAj8O6tiM6GLkra7nTgfPCdrwJ+KaZXabolssaoqsHy8zsiiL/W7ZbiS6Pvw74uZmtC/uQuZLuJzpAvLmE9eT8zIm+k0Xh4PEh4GOjjG8zbbmy4JwrRlJruE+7LdHOfPHIvVXnXP0LB1hTzOzkpGMphV9Od250vhHOJvqJzkQuTzQa59yY5mfizjnnXJ3yM3HnnHOuTnkl7pxzztUpr8Sdc1WnKD3nE6HV8Mi4HSWtVpTKcnV4HMg5V4GGuSe+00472aRJk5IOw7lE3X777X8zs52TjkNRMpJB4CIzmxzGfQd42szODI8AvdLM/qPQeny7dq7wdl2z58QlLSZ6hvCJjI14R6LnAScR5Qj+gJn9PceyHyV6bhLgdDO7sFh5kyZNYt26ddUJ3rk6JWl90jEAhKxik7JGH8mW54AvJMrmVbAS9+3aucLbdS0vp/cAR2SN+zJRTuq9iJLbv6Q/2VDRf50o4f1U4Ot+2c25hvAaMxvJb/4XSsth3/D6+/uZN28e/f39xWd2qRf391mzStzMbiDKapXpSKIjcMLr+3Isejgh6X44S1/NSw8GnHN1LOSvznkvT9JJirrhXPfkk0/GHFn8enp6uPXWW+np6Uk6FFcFcX+fcTdsK+VIfDfgzxnDj4ZxLsX6+vqYMWMGfX19xWeO2SmnnEJ7ezunnHJK0qGMdX/Vlv68dwGeyDWTmZ1vZlPMbMrOOyd+e7/mZs+ezdSpU5k9e3bSobgqiPv7TCx3upmZpIpa1SnqlP0kgAkTsnvEc3Hq7u7mmWeeobu7m66urqTD2cptt9221WsaTJ8+nYGBgc3DbW1tLFu2rMASDaGPKGf0meF1tPmsG9LkyZP5r//6r6TDcFUS9/cZ95l4KUfiG4j6aR3xOrb0irOVsXbEnmZz5sxhhx12YM6cOUmH8hJvf/vbt3pNg4GBAfouXbH5L7NCbwSSfkHUMcjekh6VdCJR5X2opD8Qdel4ZpIxOtcI4j4TL+VIfBVR37IjjdkOY0vPLy6lurq6UncGPuLss89OOoRUkXStmR0s6dvFHvEqV4G+sg+uRXnOpUV/fz89PT3Mnj2byZMn17y8mp2Jj+ZIXNIUSRcAmNnTRF3e3Rb+TgvjnHPVsYukdwFdkt4i6a2Zf0kH51w9i7thW83OxEdzJG5m64CPZwwvBhbXKDTnxrqvAQuIblWdTdTn8ggj6ivbOVeGkQZtDd+wzTmXmMfNrEPS18zstKSDca6RNHrDNudiN336dKZPn550GGnyg/D6viSDcM5Vzitx1/AGBgYarvV3hZ6XdD6wm6QfZP8lHZxLjzRnk0trbopFixYxbdo0Fi1aFEt5Xom7qkjzxu5e4r3AdcA/gdtz/DkHpDubXGZuijTp7e1l06ZN9Pb2xlKe3xN3VTGysQOpS1zRtM02bHrxRaZ3drJs+fLYy+/s6GRwaPAl47uO6dhquL29/SXztLa0snxFdWM2s78Bv5R0v5ndVdWVu4YSdyOt0ZgzZw7d3d2py00xc+ZMent7mTlzZizleSXuqiLNG/umF1/kstNP5+j584vPXAODQ4N8cdZZZS171pIvVjkakPQlM/sO8PFcWRPN7DNVL9TVpTRnk0trboq5c+cyd+7c2MrzStxVRZo3dvcS94dX7+PTuTpXtBKXdABwp5kNSToeeCvwfTNLRb/FzrnRMbMrw+uFxeZ1Y1vc2cfc6JVyJv5jYF9J+wKnABcAFwHvqWVgzrnakHQleboBBTCz9F2jdIlIc1uXkU6E0tB5UHaHRtlqGWMplfgLocexI4FzzeynIYWqc64+jeyNjwZeC/w8DB8H/DWRiFwqpbmty8DAAKsuvpjDP/zhpEPZ3KFRPtmNWKuplEp8QNJXgOOBdknbANvVLCLnXE2Z2fUAks42sykZk66U5PfJ3Wbe1iX9SnlOfCbwHHCimf2FKN9yeU1tnXNp0iLp9SMDkvYAWhKMxzk3SkXPxEPFfY6k7SXtCAwCV9U8MudcrX0eWCvpIaJOUCYCJyUbknNuNEppnf7vwDeJsjuNNIYx4PV5F3I14S1FXTWZ2UpJewFvCqN+Z2bPJRmTS5c0NR7LNL2zE2Dz/fAkEjl1dnYyOLgliVOx+96ZyZxaW1tZXqV4S7knPg+YHLI8uQSlvaUokKoNHWB6R7RhjSR6md7RwbIV+RugjDWh0o41a5ukR4ABYBNRw9kphZdwSRkYGODivov5cFfyjccyDQwOcsn8BZuHjz19YewxDA4Ocvrnf1TWsvO/+8mqxVFKJf5HYGO1CpS0N5CZVPb1wNfM7HsZ8xwIXAE8HEZd5l0mpr+laBoNDA2x5MTNXdUz66cXJBiNyzDNTwycq1wplfhXgJsk3ULUwA0oPzWjmT0A7AcgqQnYACzNMeuNZvbecspoVN5S1DnnXKZSWqf/hKjHo5upfk9HBwN/9OxvzsVP0gGSWsL74yWdI2liDEUbcLWk2yWN+YZ03gOgq0QpZ+LbmdkXalT+B4Ff5Jn2Tkl3AY8B88zs3hrF4NxYlVQ2xneb2QZJrwZWS/qdmd0wMjFU7CcBTJgwocahJC/NbV3SprOjg8GhIeCl98E7OzpYXsX2Lh0dnQzl6H0wUyX3tnP1WjiipaWVFSX2XlhKJb4ibFRXsvXl9KdLKiEPSeOALqLL9dnuACaa2aCkTuByYK8c6xhTG7tzVZZINkYz2xBen5C0FJgK3JAx/XzgfIApU6bkTQ/bKNLc1iVtBoeGWHzMMTmnnXDppVUta2hokA+8vfq9CJbiV7eVnoqllEr8uPCaWdlW4xGzDuAOM3tJmkcz+0fG++WSfiRpp+yGMGNtY3euymLPxhgu329jZgPh/WHAmG606m1dXCVKSfayR43KPo48l9IlvRb4azhLmEp07/6pGsXh3Fg1E/gQIRujpAnUPhvja4ClkiDa//y3ma2scZnONayiDdskNUuaL+n8MLyXpIpajYcj8EOByzLGzZU00pP6+4H+cE/8B8AHzczPtJ2rIjP7i5mdA9wVVzZGM3vIzPYNf282s2/Vsrx6kNaGbZ0hocrIM+Ijwy5dSrmc/jOi1ujvCsMbgEuoYGM3syHgVVnjFmW8Pxc4t9z1O5fZACb72fBqN4CpV56NMR3S2rBtcHCQU7tP3Tz8rTnxH291HnEEgxu3TlNS6N73VlnRmptZvrL8izzDw8Pc8ce1ZS9fieHh4ZLnLaUS39PMZko6DsDMNipcC3MOoKNzS7rB9vZ2WlpbWLE82UpycGiIRQcdnHPa3OuurWpZpbRiPWtJ+Q1kqtWKNQfPxpgC3rAtv8GNG/nev+xT1rKfu/++isoeN24cb93zwIrWUa4Hn76t5HlLqcSHJY0nHKlL2pOMVurODQ0OccLZJ2weXnzK4gSjiV+9tGLNoarZGF15vGGbq0Qplfg3gJXA7pIuBg4APlbLoJxzsahqNkbnXPxKaZ1+taTbgf2Juiv8rF9+c64hjGRjvAd4MeFYas57AXSNqJSuSK81s4OBZTnGOefqVy2zMaZOWhuQpa27z47ODoYGo0ah2Y3Zqt3mJVfDtWyV3Nsu1J6k0oZvaZG3Epf0cqAZ2EnSK4nOwgG2B3aLITbnXG3VJBtjWqW1AdnAwADXLV3CQUfNSjoUIGrjctTpR+WdvnR+rv6qyjO4cSMLSaad9IIiBw/1otCZ+L8DnwN2JUqDOuIf+ONfzjWCWmVjTCVvQOYaUd5K3My+D3xf0qfN7IcxxuSci0ENszE652KSN2ObpIPC2w2Sjs7+iyk+l2HhwoW0t7ezcOHC4jM7V0QtsjGmWVozozlXCeXLZirpm2b2dUk/yzHZzOyEHOMTM2XKFFu3bl3SYdRUZiONG264ocCc8chsAJNL3A1gaqVYA5j999+ffV57QIwRbXHfX37NzTffvHlY0u1mNqWUZSX1EmVj/IiZTZbUDNxkZvvVJNgyVHO7njdvHrfeeitTp05N/LL69M4OBgpsO22tLSyLKWHSEZ1HsHGw/G2rubWZlcvLayC2//77c/DzL5RddiWu3W7brbadbKUkcaqV7CROhbbrQpfTvx5e/ZnwlDj00ENZvXo1hx56aNKhAFEDmAO/emDe6WvPWFu1stLcAKZeMjvlMKayMaapYdvA4BBXfLUj7/Qjz4gv4+HGwY3sffLeZS//wLkPlL3suHHjOGzcy8pevhI3Urg7jtFkQmxvb6fv0vzfWdcxHTU78Sol2YtLiQULFrBgwYKkw3CNY0xlY/SGba4ReSXu3Nj1DTwbo3N1rWhXpM65xmRmVwNHA7OBXwBTzGxNokHVUF9fHzNmzKCvry/pUJyrmkLJXgq2QDezywpNd86l21jLxtjd3c0zzzxDd3c3XV1dsZc/veMIBoa2tK8odt87syFrW0szy1aUn13siI4j2DiUv21HJfe1IX9mtOaWZlYWiLu1uTmxpCutzc1VW1dbWxtdx+Rv49DW1la1srIVupw+I7y+mqgv8evC8DTgJqDsSlzSI8AAsAl4IbvVXWhc832gk6iXpdlmdkf2epxzo5d0NkZJRxBt303ABWZ2Zq3LBJgzZw7d3d3MmTMnjuJeYmBoI7/66OvKWvYDFz5aUdkbhzay6dhNFa2jrHIvKVxBF0t72t7eXlFXpHE9xZNkutxCrdM/BiDpamAfM3s8DO8C9FSh7GkFOlLpAPYKf+8AfhxenXOVSywbo6Qm4DzgUOBR4DZJfWZWWefPJejq6krkDNy5WiqlYdvuIxV48FdgQo3iGXEkcJFFD7HfLOkVknbJisM5V4aEszFOBR40s4cAJP2SaHuveSXuXCMqpWHbtZJWSZotaTbR/bNrKizXgKsl3R46YMi2G/DnjOFH8U5XnKuKhLMx+rbtGtqiRYuYNm0aixYtiqW8UvoTP1nSUcBIy4XzzazSbmzebWYbJL0aWC3pd2Y26psX4QDgJIAJE2p9cSBZ0zs7GRjckj2orbWVZctLT0ZQDbkaxxRL6JLZ4KVYIxcXm/cQtXGZkWOaUUF7l2qo5nY90s1nPtXs/nN6x+EMDD1bcJ5K7m0X6lazrWU8y1asyjt9m6ZtGOrJnyGuVtq2r6xBV2tzc9ldkVaz4dpo9Pb2smnTJnp7e5k7d27Nyyv1OfE7gAEzuybkW24zs/xbRhFmtiG8PiFpKdEltsxKfAOwe8bw68K47PWcD5wPUXrGcuOpBwODg6w470ubhzs+9Z3YY9g4tJGJcyeWvfz6ReurGI0rV8LZGItu29XcrgcGBlhy6VV5p886pnqp4geGnuXnh/y9ausbjeOLXBtdu2Zt3mnt7e0VZ2yrVQOyfA3f2tvbuWT+lsRXx56+MBWpqAFmzpxJb28vM2fOjKW8opfTJc0B/gf4SRi1G3B5uQVKapHUNvIeOAzI7pGgD/iIIvsDz/j9cOcawm3AXpL2kDQO+CDR9u5cQ5g7dy5r1qyJ5SwcSjsT/xTRmfItAGb2h3AZvFyvAZaGFM3bAv9tZislzQ3rXwQsJ3q87EGiR8w8i5RzDcDMXpB0MrCK6BGzxWZ2b8JhOVe3SqnEnzOz4ZF+ESRtC0UyxxcQWqXum2P8ooz3RnTw4JxrMGa2nOhA3bmG09/fT09PD7Nnz2by5Mk1L6+USvx6SV8Fxks6FPgkcGVtw3LO1UojZ2Ps6OxkaHDr7iOL3fceaTDW0trKigoaiw4PD3PJ/fEnVBkpu1zNrc0VZWxrbo2/AVlbayvHnr5wq+G06Onp4dZbbwWIpcOdUirxLwMnAvcQJYlYbmbdNY3KOVdLNcvGmLShwUFmfKG8HeeV58yrqOxx48Zx7L80VbSOcl2xYVzZy2b3Bd7e3s5Rpx+Vd/6l85cm3ogs7idzRiPuLm9LqcQ/HZJDbK64JX02jHPO1ZkYsjE6N2bF3eVtKclePppj3Owqx+Gci18S2Ridc1WUtxKXdJykK4E9JPVl/K0Bno4vxPj19/czb948+vuzn3xzrqHUIhujc7Hw/XSk0OX0m4DHgZ2AszPGDwB31zKopMXdMKGQ6Z0dDAxGmZayE7y0t7fT1trCsuWFuzQsVbHuCqHyhC3ldlk4PDzM1c+/UFHZ5RrervBdp5aWVn5121kxRfPSsstVo2yMNXV4RwfPDhXOPFbJve1CWdHGt7SwakX+ba2tZXzRpCu10tYyvmrramltYen8/D+DltaWqpVViTTtp5NUqBez9cB6SR8GHjOzfwJIGk+UZemRWCJMQNwNEwoZGBziiq/m76e2WJ/Eo5FUd4VQvMvCcePGcdi4l8UUzdZuLPJE5YoVhRvZtLe388VZ5VXyZy35Yq0bEVU1G2OtPTs0xNA7Pp5M4bdcUHByobSnEP0OKumKNK7GZCuqdFJQa2naTyeplIZtvyJqwTpiE3AJ8PaaRJQCcTdMcC4JIRvjScCOwJ5E2RgXAQcnGZdzpfD9dKSUhm3bmtnmhxDD+/KfZ3DOpcWngAOI+hHHzP5A9NiZc65OlFKJPympa2RA0pHA32oXknMuJs9lHqBXmo3RNZ6+vj5mzJhBX1/60tt7w7ZIKZfT5wIXSzqPaAN/FPhITaNyzsWh7rIxDg8PM/zgzYmUvV0FWdEA2lqay+6KtK0lmW41u7u7eeaZZ+ju7qarq6v4AjHyhm2RUvoT/yOwv6TWMDxYZBHnXH2ou2yM48aN4/k37J9M2U9Vdsa3LOPpi/b29qINVpPOigYwZ84curu7mTNnTtKhvIQ3bIsUrcQlvQY4A9jVzDok7QO808x+WvPonHO15NkYXUFdXV2pOwMf4Q3bIqXcE+8h6jZw1zD8e+BzNYonFWbNmkV7ezuzZs1KOhTnasmzMTpX50q5J76Tmf1K0ldgc3/AyTxMHJP169dv9Vpt0zsOZ2Do2ZLnL/YseKEEFdnaWsbnfZ51eHiY5377XMnrqqaXDRd+Bry1uZkFGws/S14rrc2V3Y9sbWnlrCVfLHvZapN0HPAhQjbGjEltNHg2Rjc6fX19my+np+2MPO4uP9OqlEp8SNKrCK1WJe0PPFNugZJ2By4CXhPWeX725TtJBwJXAA+HUZeZ2WnlljlaEydOZP369UycOLEm6x8YepafH/L3mqy7mEIZpcaNG0fTW5LphanpwcLlLl+ZP5sbRAcy5+zx+rLK/sLDD9X0/uPyjGQw7e3t/PBbF+ad99OnfjSOe6F1m41xfEtL0aQrNS27StpaWwoenLelJCuaN2xLv1Iq8S8AfcCekn4N7Ay8v4IyXwBOMbM7JLUBt0tabWb3Zc13o5kV7gi4RpYsWZJEsc7FIqlsjJK+AcwBngyjvmpmo+pTslDaU4gOkirpijSuxmSZqZLb29u5bukSDjpqVioas2Xyhm3pV0rr9DskvQfYGxDwgJk9X26Bodekx8P7AUn3E2WKyq7EnXO1lUQ2xu+a2dg9baoz3rAt/fJW4pKOzjPpjZIws8sqLVzSJOAtwC05Jr9T0l3AY8A8M7u30vJKMX36dAYGtqSObmtrY9myZXEU7VzcXpKNUZJnY3SujhQ6E59RYJoBFVXi4bnzS4HPmdk/sibfAUw0s0FJncDlwF451nESUe5nJkyoTjfIAwMDLLn0qs3Ds45J5Iq+c3F4UlKXmfVBbNkYT5b0EWAd0W21ZBqHuJKkufFYmmOLU6FezD5Wq0IlbUdUgV+c64w+s1I3s+WSfiRpJzP7W9Z85wPnA0yZMsXTRTo3OlXPxijpGuC1OSadCvwYWBjKWkjUqO6EHOuo+sG5K0+aG4+lObY4xZ7sRZKAnwL3m9k5eeZ5LfBXMzNJU4meZ3+qnPKcc7nVIhujmR1SynySuoGrck3zg/P0SHPjsTTHFqdSWqf3AD8jOpKGKNlLL1FFXI4DgFnAPZLuDOO+CkwAMLNFRK3fPyHpBeBZ4INm5huzc1UUdzZGSbuEhq0ARwFju+eKOpDmxmNpji1OsSd7MbP/JWrlXmiec4Fzyy3DOVeSHqp7gF7MdyTtR3Q5/RGifO3OuQrEnuwlLY7o6GTjUO6rh9mN2bIzojW3tLJyxageb93K8PAwl9yfTNK74QI9MTW3NLPxkmSyojUn1EvTGBdrNkYz8zzGzlVZEsleUmHj0CD7fuTrZS1710XfrKjscePGcey/JJMZ7YoN+Z8gWrmieFa0iXPLz2K3ftH61CWzGOMa7gC9pbWVK8+ZV/aySWhq2oaDjppFU1MpXVk4t7XYk70451Kj4Q7QVywv/wpZUg466GBWr17NQQcdnHQorg4VSvbyduDPZvaXcJntbcAxROkav2Fm3lGCc3XMD9DTYcGCBSxYsCDpMFydKnQm/hPgEABJ7cCZwKeB/Yge/6jrI3bnxqo4sjE65+JR6CZMU8bZ9kyi3sYuNbMFwBtqH5pzrkZmFPhr2BSFfX19zJgxg76+vuIzx6i/v5958+bR3+9P3LnRK3Qm3iRpWzN7ATiYkEGphOWccylWy2yMaZbWbjU985irRKEz8V8A10u6gijhyo0Akt5Anbdgdc5FyV4k/VTSijC8j6QTk46rVubMmcMOO+yQum41Z8+ezdSpU8d85jFXnkK5078l6VpgF+DqjIxp2xDdG3fO1bce4k32kqi0dqvpmcdcJQpeFjezm3OM+33twqmewzs6eHZoqOA8lTzvnZ0AJtP4lhZWrViRd3pby3iOv6bsoivS1jK+7GWbW5pZv2h9RcvXSmtzM194+KGyl41La2srnz71owWnxyjWZC/Ouepr2Hvbzw4NMfSOjydT+C0XFJy8bMWqklfV3t7OFV/tyDv9yDNWxJZAJV8ymPb2dk44e0tnVItPWRx7UpflK/Mnqmlvb+ey00/n6PnzE082szxdzzE3XLKXQhYtWkRvby8zZ85k7ty5SYfjXFV4iiDnxq7sZC8X0cC3ynp7e9m0aRO9vb1Jh+Jc1Xgl7twYZWZ3AO8B3kXUGcmbzezuZKOqnZkzZ9LU1MTMmTOTDsW5qmnYy+nOudzGajbGuXPn+mV013D8TNy5secnwDBslY3xIqL74ecnGJdzbpQSqcQlHSHpAUkPSvpyjukvk9Qbpt8iaVICYTrXqDwbo3MNIvZKXFITcB7QAewDHCdpn6zZTgT+bmZvAL4LfDveKJ1raE2SRm6lHQxclzHNb7E5V0eSOBOfCjxoZg+Z2TDwS+DIrHmOBC4M7/8HOFiSYozRuUZWs2yMko6VdK+kFyVNyZr2lXB17QFJh1dSjnMuksRR927AnzOGHwXekW+e0PDmGeBVwN9iidC5GMyaNYv169czceJElixZElu5Nc7G2A8cTXTffbNwte2DwJuBXYFrJL3RzDy5jHMVqOtLZ5JOInTMMmHChK2nNTXBdd9PIiy0/fZVW1dbawtHnlEg+1trS9XKKldLawuLT1m81XCaNG2zDUfPn09bvNnQilq/fv1Wr3GqVTZGM7sfIMeFsyOBX5rZc8DDkh4kuir3m0rLdG4sS6IS3wDsnjH8ujAu1zyPhnt3OwBPZa/IzM4ntKadMmWKZU67fs2aKoacnGXL81fgabEi5TGuWbs26RBymjhx4uYz8TFgNyDzwOHRMC42/f399PT0MHv2bCZPnhxn0c7VTBKV+G3AXpL2IKqsPwh8KGuePuCjREfp7weuy7jk51xDiPMSejVJugZ4bY5Jp5rZFVVYf94rbJXwLj9dI4q9Eg/3uE8GVgFNwGIzu1fSacA6M+sj6kVpSbjk9jRRRe+cSwEzO6SMxUq5Ajey/rxX2Cox0tWnd/npGkki98TNbDmwPGvc1zLe/xM4Nu64nHM10wf8t6RziBq27QXcGmcA3uWna0RqlKvUkp4EqtlCaCfS2Ro+rXGBx1aOasc10cx2ruL6RkXSUcAPgZ2B/wPuNLPDw7RTgROAF4DPmVnRxhS+XaeCxzZ6sW3XDVOJV5ukdWY2pfic8UprXOCxlSOtcTWqtH7eaY0LPLZyxBmX5053zjnn6pRX4s4551yd8ko8v7T25pTWuMBjK0da42pUaf280xoXeGzliC0uvyfunHPO1Sk/E3fOOefqlFfigKTFkp6Q1J81frakSUn1oCZpd0lrJN0Xeob6bIpie7mkWyXdFWL75kgskr4RXhPreU5Sk6TfSroqZXE9IukeSXdKWpcxPtHvs9FJ+rCku8Nnf5OkfZOOaYSkN0n6jaTnJM1LOp4Rko4IPc49KOnLSceTKd8+O2mF9tm14pV4pAc4YmRA0m6SLiDKMPVuYFFCcb0AnGJm+wD7A5+SdEBKYnsOOMjM9gX2I/r89pf0A2BHSe8DvpVQbACfBe4P7/dLUVwA08xsPzObkqLfWqN7GHiPmf0/YCHpupf6NPAZIDWZaCQ1AecBHcA+wHGhJ7q06CFjn50iufbZNf3c6roXs2oxsxskTcoY3hASU9xC1LViV0JxPQ48Ht4PSLofaAbSEJsBg2Fwu/D3IvAjopz325nZJ5KITdLrgOlElfUXzOy3kp5NOq5c0vJba3RmdlPG4M1EaV9TwcyeAJ6QND3pWDJMBR40s4cAJP2SqCe6+xKNKsjeZ6dFnn32btTwc/Mz8Rwk7QqcDiwGeomOSBMVfrBvAR4gJbGFS9Z3Ak8Aq4nOzj8BLAFWSTo9odC+B3yJ6KACSfulJC4AA66WdLukk9L4WxsDTgTS3fVe8nYD/pwxHHuvc/UuY599Sy3L8Uo8BzN7zMzmAH8CbgQ+mWQ8klqBS4lSVf4pLbGZ2SYz24/orGYqsMnMPgs8bWaXAwvijknSe4EnzOz2jNF3JR1Xhneb2VuJLlN+CnhDWr7PsUDSNKJK/D+SjsU1rqx99j9qWZZX4gWYWY+ZPZJkN6iStiP6MVxsZpelKbaMWP4PWAMcHoa/EV6TiO0AoEvSI8AvgYOIzsCTjotQ9obw+gSwlOjgJ1XfZ6OQ9KnQgPBOSbtK+lfgAuBIM3sqTbElGUseJfc657aWb59dK16Jp1hoqfxT4H4zOyfpeDJJ2lnSK8L78cChwO8SDQows6+Y2evMbBJRF7bXmdnxCYcFgKQWSW0j74HDiO6Duxows/NCA8L9iNr/XAbMMrPfJxvZ1rGZ2WNJx5PDbcBekvaQNI5oW+pLOKbUS2Kf7ZU4IOkXRI2e9pb0qKQTk44pOACYBRyUcdTemXRQwS7AGkl3E23wq83sqoRjSrvXAP8r6S6ibjiXmdnKhGMaK74GvAr4UfbjfUmT9FpJjwJfAOaHfdD2ScZkZi8AJwOriJ7y+JWZ3ZtkTJl8n72FZ2xzzjnn6pSfiTvnnHN1yitx55xzrk55Je6cc87VKa/EnXPOuTrllbhzzjlXp7wSd8455+qUV+LOOedcnfJK3DnnnKtT/x8QE6Qi1YpXWgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 504x158.4 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "fig, (ax, ax2) = plt.subplots(1,2,figsize=(7,2.2))\n",
    "#f = plt.figure()\n",
    "#f.set_figwidth(3.7)\n",
    "#f.set_figheight(1.9)\n",
    "\n",
    "sns.boxenplot(data = [gbr.loc[gbr.rating == 1, \"selectedCount\"],\n",
    "              gbr.loc[gbr.rating == 2, \"selectedCount\"],\n",
    "              gbr.loc[gbr.rating == 3, \"selectedCount\"],\n",
    "              gbr.loc[gbr.rating == 4, \"selectedCount\"],\n",
    "              gbr.loc[gbr.rating == 5, \"selectedCount\"]], linewidth=1, flier_kws={\"marker\":\"o\", \"s\":2}, ax=ax)\n",
    "ax.set_ylim([-0.5, 10.5])\n",
    "ax.set_xticklabels([\"1*\",\"2*\",\"3*\",\"4*\",\"5*\"])\n",
    "ax.set_ylabel(\"Selected items\")\n",
    "ax.set_title(\"Selected items vs. rating\", fontsize=10)\n",
    "\n",
    "sns.boxenplot(data = [rcd.loc[rcd.relative_comparison_numeric == 2, \"difference\"],\n",
    "              rcd.loc[rcd.relative_comparison_numeric == 1, \"difference\"],\n",
    "              rcd.loc[rcd.relative_comparison_numeric == 0, \"difference\"],\n",
    "              rcd.loc[rcd.relative_comparison_numeric == -1, \"difference\"],\n",
    "              rcd.loc[rcd.relative_comparison_numeric == -2, \"difference\"]], \n",
    "              linewidth=1, flier_kws={\"marker\":\"o\", \"s\":2}, ax=ax2)\n",
    "ax2.set_ylim([-10.5, 10.5])\n",
    "ax2.set_xticklabels([\"-2\",\"-1\",\"0\",\"1\",\"2\"])\n",
    "ax2.set_ylabel(\"Selected items diff\")\n",
    "ax2.set_title(\"Selected items vs. relative comparison\", fontsize=10)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig(\"low_high_comparison.pdf\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d33bd43c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}