BootstrapAnalysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "44d46dd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "from scipy.stats import fisher_exact, ttest_rel, ttest_ind, binom_test\n",
    "import json"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b49e1fb",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4b8ef789",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "infile = open(\"interaction_data.pkl\",'rb')\n",
    "d = pickle.load(infile)\n",
    "infile.close()\n",
    "\n",
    "infile = open(\"phase3_selections.pkl\",'rb')\n",
    "selected_item_interactions = pickle.load(infile)\n",
    "infile.close()\n",
    "\n",
    "infile = open(\"df_completed_participation.pkl\",'rb')\n",
    "df_completed_participation = pickle.load(infile)\n",
    "infile.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b5540ef",
   "metadata": {},
   "source": [
    "# Would the results be affected if we deliberately select the wrong design?\n",
    "## All selections, deliberately selecting disadvantaged / advantaged slot for one method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "42bce971",
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_item_interactions[\"selected_algorithm\"] = \"GAMMA\"\n",
    "selected_item_interactions.loc[selected_item_interactions.variant == selected_item_interactions.DELTA, \"selected_algorithm\"] = \"DELTA\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "94f9fc2f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>participation</th>\n",
       "      <th>iteration</th>\n",
       "      <th>result_layout</th>\n",
       "      <th>selected_algorithm</th>\n",
       "      <th>variant</th>\n",
       "      <th>selected_count</th>\n",
       "      <th>selected_algorithm_v1</th>\n",
       "      <th>variant_v1</th>\n",
       "      <th>selected_count_v1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>36</td>\n",
       "      <td>1.0</td>\n",
       "      <td>row-single-scrollable</td>\n",
       "      <td>DELTA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>GAMMA</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>36</td>\n",
       "      <td>2.0</td>\n",
       "      <td>columns</td>\n",
       "      <td>DELTA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GAMMA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>36</td>\n",
       "      <td>3.0</td>\n",
       "      <td>max-columns</td>\n",
       "      <td>DELTA</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>GAMMA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>36</td>\n",
       "      <td>4.0</td>\n",
       "      <td>rows</td>\n",
       "      <td>DELTA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>GAMMA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>36</td>\n",
       "      <td>5.0</td>\n",
       "      <td>row-single-scrollable</td>\n",
       "      <td>DELTA</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>GAMMA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1044</th>\n",
       "      <td>261</td>\n",
       "      <td>4.0</td>\n",
       "      <td>max-columns</td>\n",
       "      <td>DELTA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>GAMMA</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1045</th>\n",
       "      <td>261</td>\n",
       "      <td>5.0</td>\n",
       "      <td>rows</td>\n",
       "      <td>DELTA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>GAMMA</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1046</th>\n",
       "      <td>261</td>\n",
       "      <td>6.0</td>\n",
       "      <td>row-single-scrollable</td>\n",
       "      <td>DELTA</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>GAMMA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1047</th>\n",
       "      <td>261</td>\n",
       "      <td>7.0</td>\n",
       "      <td>columns</td>\n",
       "      <td>DELTA</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>GAMMA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1048</th>\n",
       "      <td>261</td>\n",
       "      <td>8.0</td>\n",
       "      <td>max-columns</td>\n",
       "      <td>DELTA</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>GAMMA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1049 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      participation  iteration          result_layout selected_algorithm  \\\n",
       "0                36        1.0  row-single-scrollable              DELTA   \n",
       "1                36        2.0                columns              DELTA   \n",
       "2                36        3.0            max-columns              DELTA   \n",
       "3                36        4.0                   rows              DELTA   \n",
       "4                36        5.0  row-single-scrollable              DELTA   \n",
       "...             ...        ...                    ...                ...   \n",
       "1044            261        4.0            max-columns              DELTA   \n",
       "1045            261        5.0                   rows              DELTA   \n",
       "1046            261        6.0  row-single-scrollable              DELTA   \n",
       "1047            261        7.0                columns              DELTA   \n",
       "1048            261        8.0            max-columns              DELTA   \n",
       "\n",
       "      variant  selected_count selected_algorithm_v1  variant_v1  \\\n",
       "0         0.0             4.0                 GAMMA         1.0   \n",
       "1         0.0             0.0                 GAMMA         0.0   \n",
       "2         1.0             1.0                 GAMMA         0.0   \n",
       "3         0.0             4.0                 GAMMA         0.0   \n",
       "4         1.0             1.0                 GAMMA         0.0   \n",
       "...       ...             ...                   ...         ...   \n",
       "1044      0.0             3.0                 GAMMA         1.0   \n",
       "1045      0.0             1.0                 GAMMA         1.0   \n",
       "1046      1.0             2.0                 GAMMA         0.0   \n",
       "1047      1.0             3.0                 GAMMA         0.0   \n",
       "1048      1.0             1.0                 GAMMA         0.0   \n",
       "\n",
       "      selected_count_v1  \n",
       "0                   2.0  \n",
       "1                   5.0  \n",
       "2                   1.0  \n",
       "3                   0.0  \n",
       "4                   1.0  \n",
       "...                 ...  \n",
       "1044                3.0  \n",
       "1045                2.0  \n",
       "1046                4.0  \n",
       "1047                2.0  \n",
       "1048                2.0  \n",
       "\n",
       "[1049 rows x 9 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_with_alg = selected_item_interactions.groupby([\"participation\",\"iteration\",\"result_layout\",\"selected_algorithm\",\"variant\"])[[\"DELTA\"]].count()\n",
    "selected_with_alg.columns = [\"selected_count\"]\n",
    "selected_with_alg = selected_with_alg.reset_index([3,4])\n",
    "selected_with_alg.head()\n",
    "\n",
    "v0 = selected_with_alg.loc[selected_with_alg.selected_algorithm==\"DELTA\"]\n",
    "v1 = selected_with_alg.loc[selected_with_alg.selected_algorithm==\"GAMMA\"]\n",
    "variant_joined = v0.join(v1, how=\"outer\", rsuffix=\"_v1\")\n",
    "variant_joined[\"selected_algorithm\"] = \"DELTA\"\n",
    "variant_joined[\"selected_algorithm_v1\"] = \"GAMMA\"\n",
    "variant_joined.fillna(0, inplace=True)\n",
    "variant_joined = variant_joined.reset_index()\n",
    "variant_joined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ff040173",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.724499523355577 2.742612011439466\n",
      "Ttest_relResult(statistic=11.768579083194775, pvalue=4.0561124250320255e-30)\n"
     ]
    }
   ],
   "source": [
    "dx = variant_joined[\"selected_count\"]\n",
    "dy = variant_joined[\"selected_count_v1\"]\n",
    "print(dx.mean(), dy.mean())\n",
    "print(ttest_rel(dx,dy))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c254fb0a",
   "metadata": {},
   "source": [
    "- Delta significantly outperformed gamma with p close to 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f48fa1ba",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "variant 0.0\n",
      "3.7795698924731185 2.467741935483871\n",
      "Ttest_relResult(statistic=11.124798770556337, pvalue=4.238618584024765e-26)\n",
      "\n",
      "variant 1.0\n",
      "3.6619144602851326 3.054989816700611\n",
      "Ttest_relResult(statistic=5.261644045051465, pvalue=2.138461991416444e-07)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in variant_joined.variant.unique():\n",
    "    dx = variant_joined.loc[variant_joined.variant == i,\"selected_count\"]\n",
    "    dy = variant_joined.loc[variant_joined.variant == i,\"selected_count_v1\"]\n",
    "    print(\"variant\", i)\n",
    "    print(dx.mean(), dy.mean())\n",
    "    print(ttest_rel(dx,dy))\n",
    "    print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "82f33db0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "variant 0.0 row-single-scrollable\n",
      "3.206896551724138 2.2988505747126435\n",
      "Ttest_relResult(statistic=3.2691083511886783, pvalue=0.0015522311675777935)\n",
      "\n",
      "variant 0.0 columns\n",
      "3.63265306122449 2.5510204081632653\n",
      "Ttest_relResult(statistic=3.8473210962379505, pvalue=0.0002138042916200897)\n",
      "\n",
      "variant 0.0 max-columns\n",
      "4.089887640449438 2.5280898876404496\n",
      "Ttest_relResult(statistic=4.537411622639667, pvalue=1.7935587853285236e-05)\n",
      "\n",
      "variant 0.0 rows\n",
      "3.956989247311828 2.6774193548387095\n",
      "Ttest_relResult(statistic=4.639792852673591, pvalue=1.1522773256267972e-05)\n",
      "\n",
      "variant 0.0 column-single\n",
      "3.784313725490196 2.588235294117647\n",
      "Ttest_relResult(statistic=4.213673365089623, pvalue=5.4738500370854116e-05)\n",
      "\n",
      "variant 0.0 row-single\n",
      "4.0 2.1235955056179776\n",
      "Ttest_relResult(statistic=7.215596366342409, pvalue=1.7959817891912334e-10)\n",
      "\n",
      "variant 1.0 row-single-scrollable\n",
      "3.546875 3.0\n",
      "Ttest_relResult(statistic=1.9477392655515526, pvalue=0.05590532427626266)\n",
      "\n",
      "variant 1.0 columns\n",
      "3.7093023255813953 2.941860465116279\n",
      "Ttest_relResult(statistic=2.9422096754386544, pvalue=0.004199091760177967)\n",
      "\n",
      "variant 1.0 max-columns\n",
      "3.876543209876543 2.753086419753086\n",
      "Ttest_relResult(statistic=4.194951719991616, pvalue=7.008088678325968e-05)\n",
      "\n",
      "variant 1.0 rows\n",
      "3.5058823529411764 3.5176470588235293\n",
      "Ttest_relResult(statistic=-0.042427619106199486, pvalue=0.9662585004168536)\n",
      "\n",
      "variant 1.0 column-single\n",
      "3.6451612903225805 3.10752688172043\n",
      "Ttest_relResult(statistic=1.8916738215708866, pvalue=0.06168006307293002)\n",
      "\n",
      "variant 1.0 row-single\n",
      "3.6707317073170733 2.975609756097561\n",
      "Ttest_relResult(statistic=2.269000994801043, pvalue=0.02592803498003375)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in variant_joined.variant.unique():\n",
    "    for j in variant_joined.result_layout.unique():\n",
    "        dx = variant_joined.loc[((variant_joined.variant == i)&(variant_joined.result_layout==j)),\"selected_count\"]\n",
    "        dy = variant_joined.loc[((variant_joined.variant == i)&(variant_joined.result_layout==j)),\"selected_count_v1\"]\n",
    "        print(\"variant\", i,j)\n",
    "        print(dx.mean(), dy.mean())\n",
    "        print(ttest_rel(dx,dy))\n",
    "        print(\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78d5f592",
   "metadata": {},
   "source": [
    "- considering only the data, where Delta was in disadvantaged slot and rows:double layout, we would be unable to confirm its supremacy over gamma"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21836eb7",
   "metadata": {},
   "source": [
    "# All selections, bootstraping analysis: \n",
    "## how many participants to get significant results\n",
    "### within subject design (all participants has both ordering of methods)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "0819cf90",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(6828, 13)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rt1 = d.drop_duplicates([\"participation\",\"iteration\",\"DELTA\"]).set_index([\"participation\",\"iteration\",\"DELTA\"])[[\"result_layout\"]]\n",
    "sii = rt1.join(selected_item_interactions.set_index([\"participation\",\"iteration\",\"DELTA\"]), how=\"outer\", rsuffix=\"_r\")\n",
    "sii = sii.reset_index().fillna(0)\n",
    "sii.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "8d2785e3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "column-single 0.979 670\n",
      "columns 0.989 727\n",
      "max-columns 0.998 932\n",
      "row-single 1.0 924\n",
      "row-single-scrollable 0.989 582\n",
      "rows 0.95 454\n"
     ]
    }
   ],
   "source": [
    "contingency_table = pd.crosstab(sii.result_layout,sii.selected_algorithm)\n",
    "participants_volume = 20\n",
    "np.random.seed(42)\n",
    "res = {}\n",
    "for i in contingency_table.index.unique():\n",
    "    resAgg = []\n",
    "    resAggSign = []\n",
    "    participantsPool = sii.loc[sii.result_layout == i,\"participation\"].unique()\n",
    "    for j in range(1000):                \n",
    "        participants = np.random.choice(participantsPool, participants_volume, True)\n",
    "        results = sii.loc[sii.result_layout == i].set_index(\"participation\").loc[participants]\n",
    "        groups = results.groupby(\"selected_algorithm\")[[\"movieID\"]].count()\n",
    "        resAgg.append(groups.loc[\"DELTA\",\"movieID\"] >= groups.loc[\"GAMMA\",\"movieID\"])\n",
    "        \n",
    "        succ = groups.loc[\"DELTA\",\"movieID\"]\n",
    "        fail = groups.loc[\"GAMMA\",\"movieID\"]\n",
    "        bt = binom_test(succ, succ+fail,p=0.5,alternative='greater')\n",
    "        resAggSign.append(bt)        \n",
    "        \n",
    "    print(i, np.mean(resAgg), np.sum(np.array(resAggSign)<=0.05))\n",
    "    res[i] = [np.mean(resAgg), np.sum(np.array(resAggSign)<=0.05)]\n",
    "resultsWS = pd.DataFrame(res).T"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e34905be",
   "metadata": {},
   "source": [
    "### between-subject design: each participant is assigned to only one ordering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "b703edf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# using only the first occurence per user\n",
    "sii4 = sii.loc[sii.iteration <=4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "a9379eb0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "column-single 0.988 862\n",
      "columns 0.971 525\n",
      "max-columns 0.997 913\n",
      "row-single 0.985 772\n",
      "row-single-scrollable 0.975 507\n",
      "rows 0.858 330\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "res = {}\n",
    "for i in contingency_table.index.unique():\n",
    "    resAgg = []\n",
    "    resAggSign = []    \n",
    "    participantsPool = sii4.loc[sii4.result_layout == i,[\"participation\",\"DELTA\"]].drop_duplicates()\n",
    "    for j in range(1000):                \n",
    "        participantsIDs = np.random.choice(participantsPool.index, participants_volume*2, True)\n",
    "\n",
    "        results = sii4.loc[sii4.result_layout == i].set_index([\"participation\",\"DELTA\"]).loc[participantsPool.loc[participantsIDs].values.tolist()]\n",
    "        groups = results.groupby(\"selected_algorithm\")[[\"movieID\"]].count()\n",
    "        resAgg.append(groups.loc[\"DELTA\"] >= groups.loc[\"GAMMA\"])\n",
    "        \n",
    "        succ = groups.loc[\"DELTA\",\"movieID\"]\n",
    "        fail = groups.loc[\"GAMMA\",\"movieID\"]\n",
    "        bt = binom_test(succ, succ+fail,p=0.5,alternative='greater')\n",
    "        resAggSign.append(bt)  \n",
    "        \n",
    "    print(i, np.mean(resAgg), np.sum(np.array(resAggSign)<=0.05))\n",
    "    res[i] = [np.mean(resAgg), np.sum(np.array(resAggSign)<=0.05)]\n",
    "resultsBS = pd.DataFrame(res).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "7ae02249",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>column-single</th>\n",
       "      <td>-0.009</td>\n",
       "      <td>-192.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>columns</th>\n",
       "      <td>0.018</td>\n",
       "      <td>202.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max-columns</th>\n",
       "      <td>0.001</td>\n",
       "      <td>19.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row-single</th>\n",
       "      <td>0.015</td>\n",
       "      <td>152.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row-single-scrollable</th>\n",
       "      <td>0.014</td>\n",
       "      <td>75.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rows</th>\n",
       "      <td>0.092</td>\n",
       "      <td>124.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           0      1\n",
       "column-single         -0.009 -192.0\n",
       "columns                0.018  202.0\n",
       "max-columns            0.001   19.0\n",
       "row-single             0.015  152.0\n",
       "row-single-scrollable  0.014   75.0\n",
       "rows                   0.092  124.0"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resultsWS - resultsBS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "40ce8154",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1.3408938505889165, 9.986574183045696e-14)"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "totalVol = 6000\n",
    "contingencyTable = [[resultsWS.sum()[1], totalVol - resultsWS.sum()[1]],[resultsBS.sum()[1], totalVol - resultsBS.sum()[1]]]\n",
    "fisher_exact(contingencyTable)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "c287640f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2.433342146288986, 6.180018853346323e-14)"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contingencyTable = [[resultsWS.sum()[0]*1000, totalVol - resultsWS.sum()[0]*1000],[resultsBS.sum()[0]*1000, totalVol - resultsBS.sum()[0]*1000]]\n",
    "fisher_exact(contingencyTable)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "31c347b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "column-single (2.4417300760913094, 2.2348199224345637e-22)\n",
      "columns (3.202643171806167, 1.2228111145914884e-35)\n",
      "max-columns (16.483285825343355, 3.865802896207846e-130)\n",
      "row-single (14.621609088801298, 4.5442513720596355e-124)\n",
      "row-single-scrollable (1.6744936028497355, 1.2645251242954949e-08)\n"
     ]
    }
   ],
   "source": [
    "for i in resultsWS.index.unique():\n",
    "    if i != \"rows\":\n",
    "        contingencyTable = [[resultsWS.loc[i,1], 1000-resultsWS.loc[i,1]],[resultsWS.loc[\"rows\",1], 1000-resultsWS.loc[\"rows\",1]]]\n",
    "        print(i,fisher_exact(contingencyTable))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "2329fca2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>column-single</th>\n",
       "      <td>0.979</td>\n",
       "      <td>670.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>columns</th>\n",
       "      <td>0.989</td>\n",
       "      <td>727.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max-columns</th>\n",
       "      <td>0.998</td>\n",
       "      <td>932.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row-single</th>\n",
       "      <td>1.000</td>\n",
       "      <td>924.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row-single-scrollable</th>\n",
       "      <td>0.989</td>\n",
       "      <td>582.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rows</th>\n",
       "      <td>0.950</td>\n",
       "      <td>454.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           0      1\n",
       "column-single          0.979  670.0\n",
       "columns                0.989  727.0\n",
       "max-columns            0.998  932.0\n",
       "row-single             1.000  924.0\n",
       "row-single-scrollable  0.989  582.0\n",
       "rows                   0.950  454.0"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "resultsWS"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63d375b5",
   "metadata": {},
   "source": [
    "### Results\n",
    "- for both the volume of concordant results and the volume of stat. sign. concordant results, we observed that within subject design gives more clear results than between-subject, even if the volume of its participants doubles\n",
    "- the rows:double has smallest chance to generate stat. sign results, e.g. below 50% for 20 participants"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "926308d7",
   "metadata": {},
   "source": [
    "# First selections, bootstraping analysis: \n",
    "## how many participants to get significant results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "8f49b1dd",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-79-09193f7f5e0a>:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  first_selections[\"selected_algorithm\"] = \"GAMMA\"\n",
      "C:\\Users\\lpesk\\AppData\\Roaming\\Python\\Python38\\site-packages\\pandas\\core\\indexing.py:1817: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_single_column(loc, value, pi)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>participation</th>\n",
       "      <th>interaction_type</th>\n",
       "      <th>time</th>\n",
       "      <th>data</th>\n",
       "      <th>iteration</th>\n",
       "      <th>result_layout</th>\n",
       "      <th>GAMMA</th>\n",
       "      <th>DELTA</th>\n",
       "      <th>variant</th>\n",
       "      <th>layout_high_level</th>\n",
       "      <th>movieID</th>\n",
       "      <th>selected_algorithm</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3271</th>\n",
       "      <td>36</td>\n",
       "      <td>selected-item</td>\n",
       "      <td>2023-01-15 23:15:59.742434</td>\n",
       "      <td>{\"selected_item\": {\"genres\": [\"Adventure\", \"Fa...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>row-single-scrollable</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>rows</td>\n",
       "      <td>106489</td>\n",
       "      <td>DELTA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3298</th>\n",
       "      <td>36</td>\n",
       "      <td>selected-item</td>\n",
       "      <td>2023-01-15 23:16:38.416542</td>\n",
       "      <td>{\"selected_item\": {\"genres\": [\"Adventure\", \"Fa...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>columns</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>cols</td>\n",
       "      <td>118696</td>\n",
       "      <td>GAMMA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3314</th>\n",
       "      <td>36</td>\n",
       "      <td>selected-item</td>\n",
       "      <td>2023-01-15 23:17:09.189259</td>\n",
       "      <td>{\"selected_item\": {\"genres\": [\"Drama\", \"Thrill...</td>\n",
       "      <td>3.0</td>\n",
       "      <td>max-columns</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>cols</td>\n",
       "      <td>116797</td>\n",
       "      <td>DELTA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3325</th>\n",
       "      <td>36</td>\n",
       "      <td>selected-item</td>\n",
       "      <td>2023-01-15 23:17:30.629024</td>\n",
       "      <td>{\"selected_item\": {\"genres\": [\"Action\", \"Myste...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>rows</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>rows</td>\n",
       "      <td>5418</td>\n",
       "      <td>DELTA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3345</th>\n",
       "      <td>36</td>\n",
       "      <td>selected-item</td>\n",
       "      <td>2023-01-15 23:18:10.680798</td>\n",
       "      <td>{\"selected_item\": {\"genres\": [\"Action\", \"Crime...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>row-single-scrollable</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>rows</td>\n",
       "      <td>168248</td>\n",
       "      <td>GAMMA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      participation interaction_type                        time  \\\n",
       "id                                                                 \n",
       "3271             36    selected-item  2023-01-15 23:15:59.742434   \n",
       "3298             36    selected-item  2023-01-15 23:16:38.416542   \n",
       "3314             36    selected-item  2023-01-15 23:17:09.189259   \n",
       "3325             36    selected-item  2023-01-15 23:17:30.629024   \n",
       "3345             36    selected-item  2023-01-15 23:18:10.680798   \n",
       "\n",
       "                                                   data  iteration  \\\n",
       "id                                                                   \n",
       "3271  {\"selected_item\": {\"genres\": [\"Adventure\", \"Fa...        1.0   \n",
       "3298  {\"selected_item\": {\"genres\": [\"Adventure\", \"Fa...        2.0   \n",
       "3314  {\"selected_item\": {\"genres\": [\"Drama\", \"Thrill...        3.0   \n",
       "3325  {\"selected_item\": {\"genres\": [\"Action\", \"Myste...        4.0   \n",
       "3345  {\"selected_item\": {\"genres\": [\"Action\", \"Crime...        5.0   \n",
       "\n",
       "              result_layout  GAMMA  DELTA  variant layout_high_level  movieID  \\\n",
       "id                                                                              \n",
       "3271  row-single-scrollable    1.0    0.0        0              rows   106489   \n",
       "3298                columns    0.0    1.0        0              cols   118696   \n",
       "3314            max-columns    0.0    1.0        1              cols   116797   \n",
       "3325                   rows    1.0    0.0        0              rows     5418   \n",
       "3345  row-single-scrollable    0.0    1.0        0              rows   168248   \n",
       "\n",
       "     selected_algorithm  \n",
       "id                       \n",
       "3271              DELTA  \n",
       "3298              GAMMA  \n",
       "3314              DELTA  \n",
       "3325              DELTA  \n",
       "3345              GAMMA  "
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_item_interactions = selected_item_interactions.sort_values([\"participation\",\"iteration\",\"result_layout\",\"time\"])\n",
    "first_selections = selected_item_interactions.drop_duplicates([\"participation\",\"iteration\"])\n",
    "\n",
    "first_selections[\"selected_algorithm\"] = \"GAMMA\"\n",
    "first_selections.loc[first_selections.variant == first_selections.DELTA, \"selected_algorithm\"] = \"DELTA\"\n",
    "first_selections.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "12964840",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>selected_algorithm</th>\n",
       "      <th>DELTA</th>\n",
       "      <th>GAMMA</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>result_layout</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>column-single</th>\n",
       "      <td>101</td>\n",
       "      <td>94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>columns</th>\n",
       "      <td>101</td>\n",
       "      <td>83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max-columns</th>\n",
       "      <td>92</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row-single</th>\n",
       "      <td>90</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row-single-scrollable</th>\n",
       "      <td>82</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rows</th>\n",
       "      <td>90</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "selected_algorithm     DELTA  GAMMA\n",
       "result_layout                      \n",
       "column-single            101     94\n",
       "columns                  101     83\n",
       "max-columns               92     78\n",
       "row-single                90     80\n",
       "row-single-scrollable     82     69\n",
       "rows                      90     86"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contingency_table = pd.crosstab(first_selections.result_layout,first_selections.selected_algorithm)\n",
    "contingency_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "526e053a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.02220170664276684\n"
     ]
    }
   ],
   "source": [
    "succ = contingency_table[\"DELTA\"].sum()\n",
    "fail = contingency_table[\"GAMMA\"].sum()\n",
    "print(binom_test(succ, succ+fail,p=0.5,alternative='greater'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "ff030aee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(556, 490)"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "succ,fail"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "9c1dd134",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((523, 12), (523, 12))"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "deltaAdvantage = first_selections.loc[first_selections.DELTA==0]\n",
    "gammaAdvantage = first_selections.loc[first_selections.DELTA==1]\n",
    "deltaAdvantage.shape, gammaAdvantage.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "c1185e73",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>selected_algorithm</th>\n",
       "      <th>DELTA</th>\n",
       "      <th>GAMMA</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>result_layout</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>column-single</th>\n",
       "      <td>77</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>columns</th>\n",
       "      <td>75</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max-columns</th>\n",
       "      <td>72</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row-single</th>\n",
       "      <td>75</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row-single-scrollable</th>\n",
       "      <td>68</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rows</th>\n",
       "      <td>81</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "selected_algorithm     DELTA  GAMMA\n",
       "result_layout                      \n",
       "column-single             77     21\n",
       "columns                   75     16\n",
       "max-columns               72     13\n",
       "row-single                75      9\n",
       "row-single-scrollable     68      9\n",
       "rows                      81      7"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contingency_table = pd.crosstab(deltaAdvantage.result_layout,deltaAdvantage.selected_algorithm)\n",
    "contingency_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "32076e66",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>selected_algorithm</th>\n",
       "      <th>DELTA</th>\n",
       "      <th>GAMMA</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>result_layout</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>column-single</th>\n",
       "      <td>24</td>\n",
       "      <td>73</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>columns</th>\n",
       "      <td>26</td>\n",
       "      <td>67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max-columns</th>\n",
       "      <td>20</td>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row-single</th>\n",
       "      <td>15</td>\n",
       "      <td>71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row-single-scrollable</th>\n",
       "      <td>14</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rows</th>\n",
       "      <td>9</td>\n",
       "      <td>79</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "selected_algorithm     DELTA  GAMMA\n",
       "result_layout                      \n",
       "column-single             24     73\n",
       "columns                   26     67\n",
       "max-columns               20     65\n",
       "row-single                15     71\n",
       "row-single-scrollable     14     60\n",
       "rows                       9     79"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contingency_table = pd.crosstab(gammaAdvantage.result_layout,gammaAdvantage.selected_algorithm)\n",
    "contingency_table"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "066368ce",
   "metadata": {},
   "source": [
    "### Within subject"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "b682b3bf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "column-single 0.812 104\n",
      "columns 0.991 620\n",
      "max-columns 0.981 462\n",
      "row-single 0.957 214\n",
      "row-single-scrollable 0.999 492\n",
      "rows 0.803 6\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "participants_volume = 200\n",
    "res = {}\n",
    "np.random.seed(42)\n",
    "for i in contingency_table.index.unique():\n",
    "    resAgg = []\n",
    "    resAggSign = []    \n",
    "    participantsPool = first_selections.loc[first_selections.result_layout == i,\"participation\"].unique()\n",
    "    for j in range(1000):                \n",
    "        participants = np.random.choice(participantsPool, participants_volume, True)\n",
    "        results = first_selections.loc[first_selections.result_layout == i].set_index(\"participation\").loc[participants]\n",
    "        groups = results.groupby(\"selected_algorithm\")[[\"movieID\"]].count()\n",
    "        resAgg.append(groups.loc[\"DELTA\"] >= groups.loc[\"GAMMA\"])\n",
    "        \n",
    "        succ = groups.loc[\"DELTA\",\"movieID\"]\n",
    "        fail = groups.loc[\"GAMMA\",\"movieID\"]\n",
    "        bt = binom_test(succ, succ+fail,p=0.5,alternative='greater')\n",
    "        resAggSign.append(bt)  \n",
    "        \n",
    "    print(i, np.mean(resAgg), np.sum(np.array(resAggSign)<=0.05))\n",
    "    res[i] = [np.mean(resAgg), np.sum(np.array(resAggSign)<=0.05)]\n",
    "resultsWS = pd.DataFrame(res).T    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae8161da",
   "metadata": {},
   "source": [
    "### Between subject"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "id": "e185ecd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# using only the first occurence per user\n",
    "fi4 = first_selections.loc[first_selections.iteration <=4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "id": "5eadcf89",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "column-single 0.987 695\n",
      "columns 0.968 606\n",
      "max-columns 1.0 979\n",
      "row-single 0.98 700\n",
      "row-single-scrollable 0.509 41\n",
      "rows 0.003 0\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "res = {}\n",
    "for i in contingency_table.index.unique():\n",
    "    resAgg = []\n",
    "    resAggSign = []    \n",
    "    participantsPool = fi4.loc[fi4.result_layout == i,[\"participation\",\"DELTA\"]].drop_duplicates()\n",
    "    for j in range(1000):                \n",
    "        participantsIDs = np.random.choice(participantsPool.index, participants_volume*2, True)\n",
    "\n",
    "        results = fi4.loc[fi4.result_layout == i].set_index([\"participation\",\"DELTA\"]).loc[participantsPool.loc[participantsIDs].values.tolist()]\n",
    "        groups = results.groupby(\"selected_algorithm\")[[\"movieID\"]].count()\n",
    "        resAgg.append(groups.loc[\"DELTA\"] >= groups.loc[\"GAMMA\"])\n",
    "        \n",
    "        succ = groups.loc[\"DELTA\",\"movieID\"]\n",
    "        fail = groups.loc[\"GAMMA\",\"movieID\"]\n",
    "        bt = binom_test(succ, succ+fail,p=0.5,alternative='greater')\n",
    "        resAggSign.append(bt)  \n",
    "        \n",
    "    print(i, np.mean(resAgg), np.sum(np.array(resAggSign)<=0.05))\n",
    "    res[i] = [np.mean(resAgg), np.sum(np.array(resAggSign)<=0.05)]\n",
    "resultsBS = pd.DataFrame(res).T"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37e202c2",
   "metadata": {},
   "source": [
    "## First three selected items"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "cdb2d6f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "selected_item_interactions[\"selectionOrder\"] = selected_item_interactions.groupby([\"participation\",\"iteration\"])[\"time\"].rank(method=\"dense\", ascending=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "3a895add",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3013, 14)"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "firstThreeSelected = selected_item_interactions.loc[selected_item_interactions.selectionOrder <=3]\n",
    "f3i = rt1.join(firstThreeSelected.set_index([\"participation\",\"iteration\",\"DELTA\"]), how=\"outer\", rsuffix=\"_r\")\n",
    "f3i = f3i.reset_index().fillna(0)\n",
    "f3i.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "e6207e93",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>selected_algorithm</th>\n",
       "      <th>DELTA</th>\n",
       "      <th>GAMMA</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>result_layout</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>column-single</th>\n",
       "      <td>312</td>\n",
       "      <td>241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>columns</th>\n",
       "      <td>295</td>\n",
       "      <td>231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max-columns</th>\n",
       "      <td>284</td>\n",
       "      <td>203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row-single</th>\n",
       "      <td>285</td>\n",
       "      <td>192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>row-single-scrollable</th>\n",
       "      <td>237</td>\n",
       "      <td>177</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rows</th>\n",
       "      <td>268</td>\n",
       "      <td>242</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "selected_algorithm     DELTA  GAMMA\n",
       "result_layout                      \n",
       "column-single            312    241\n",
       "columns                  295    231\n",
       "max-columns              284    203\n",
       "row-single               285    192\n",
       "row-single-scrollable    237    177\n",
       "rows                     268    242"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contingency_table = pd.crosstab(firstThreeSelected.result_layout,firstThreeSelected.selected_algorithm)\n",
    "contingency_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "5570e827",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.1812724736309596e-13\n"
     ]
    }
   ],
   "source": [
    "succ = contingency_table[\"DELTA\"].sum()\n",
    "fail = contingency_table[\"GAMMA\"].sum()\n",
    "print(binom_test(succ, succ+fail,p=0.5,alternative='greater'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "f8ee2a6e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1681, 1286)"
      ]
     },
     "execution_count": 112,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "succ,fail"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20c0a375",
   "metadata": {},
   "source": [
    "### Within subject"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "id": "4aeb39ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "column-single 0.997 684\n",
      "columns 0.957 597\n",
      "max-columns 1.0 844\n",
      "row-single 0.997 907\n",
      "row-single-scrollable 0.99 747\n",
      "rows 0.823 229\n"
     ]
    }
   ],
   "source": [
    "participants_volume = 50\n",
    "np.random.seed(42)\n",
    "res = {}\n",
    "for i in contingency_table.index.unique():\n",
    "    resAgg = []\n",
    "    resAggSign = []\n",
    "    participantsPool = f3i.loc[f3i.result_layout == i,\"participation\"].unique()\n",
    "    for j in range(1000):                \n",
    "        participants = np.random.choice(participantsPool, participants_volume, True)\n",
    "        results = f3i.loc[f3i.result_layout == i].set_index(\"participation\").loc[participants]\n",
    "        groups = results.groupby(\"selected_algorithm\")[[\"movieID\"]].count()\n",
    "        resAgg.append(groups.loc[\"DELTA\",\"movieID\"] >= groups.loc[\"GAMMA\",\"movieID\"])\n",
    "        \n",
    "        succ = groups.loc[\"DELTA\",\"movieID\"]\n",
    "        fail = groups.loc[\"GAMMA\",\"movieID\"]\n",
    "        bt = binom_test(succ, succ+fail,p=0.5,alternative='greater')\n",
    "        resAggSign.append(bt)        \n",
    "        \n",
    "    print(i, np.mean(resAgg), np.sum(np.array(resAggSign)<=0.05))\n",
    "    res[i] = [np.mean(resAgg), np.sum(np.array(resAggSign)<=0.05)]\n",
    "resultsWS = pd.DataFrame(res).T"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57c990b8",
   "metadata": {},
   "source": [
    "### Between subject"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "ad12db02",
   "metadata": {},
   "outputs": [],
   "source": [
    "# using only the first occurence per user\n",
    "f3i4 = f3i.loc[f3i.iteration <=4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "266af133",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "column-single 0.937 588\n",
      "columns 0.99 840\n",
      "max-columns 1.0 987\n",
      "row-single 0.996 928\n",
      "row-single-scrollable 0.834 432\n",
      "rows 0.273 33\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "res = {}\n",
    "for i in contingency_table.index.unique():\n",
    "    resAgg = []\n",
    "    resAggSign = []    \n",
    "    participantsPool = f3i4.loc[f3i4.result_layout == i,[\"participation\",\"DELTA\"]].drop_duplicates()\n",
    "    for j in range(1000):                \n",
    "        participantsIDs = np.random.choice(participantsPool.index, participants_volume*2, True)\n",
    "\n",
    "        results = f3i4.loc[f3i4.result_layout == i].set_index([\"participation\",\"DELTA\"]).loc[participantsPool.loc[participantsIDs].values.tolist()]\n",
    "        groups = results.groupby(\"selected_algorithm\")[[\"movieID\"]].count()\n",
    "        resAgg.append(groups.loc[\"DELTA\"] >= groups.loc[\"GAMMA\"])\n",
    "        \n",
    "        succ = groups.loc[\"DELTA\",\"movieID\"]\n",
    "        fail = groups.loc[\"GAMMA\",\"movieID\"]\n",
    "        bt = binom_test(succ, succ+fail,p=0.5,alternative='greater')\n",
    "        resAggSign.append(bt)  \n",
    "        \n",
    "    print(i, np.mean(resAgg), np.sum(np.array(resAggSign)<=0.05))\n",
    "    res[i] = [np.mean(resAgg), np.sum(np.array(resAggSign)<=0.05)]\n",
    "resultsBS = pd.DataFrame(res).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53f91334",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc715a9e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}