09-Task-Evaluate_Data/01d7_PRE_VIS_Gaze_Selection_Fixation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Incorporate results of manual accuracy evaluation to select fixations\n",
    "Status: Implemented\n",
    " -- Fully automatized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "from tqdm.notebook import tqdm\n",
    "from utils.utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# get all files from participants\n",
    "folders = get_participant_folder_per_participant()\n",
    "sequence_orders = get_sequence_order_per_participant()\n",
    "assert (set(folders.keys()) == set(sequence_orders.keys()))\n",
    "snippets = {p: sequence_orders[p][SNIPPET].to_list()\n",
    "            for p in sequence_orders}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fixation correction iteration: 4\n",
      "Previous Fixation correction iteration: 3\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'003': {},\n",
       " '004': {},\n",
       " '005': {},\n",
       " '007': {},\n",
       " '008': {},\n",
       " '009': {},\n",
       " '010': {},\n",
       " '011': {},\n",
       " '012': {'v0': 100, 'v1': 50, 'v2': 80},\n",
       " '013': {'v0': 20},\n",
       " '014': {},\n",
       " '015': {},\n",
       " '016': {},\n",
       " '017': {},\n",
       " '018': {},\n",
       " '019': {},\n",
       " '020': {},\n",
       " '021': {},\n",
       " '022': {},\n",
       " '023': {},\n",
       " '024': {},\n",
       " '101': {},\n",
       " '102': {},\n",
       " '106': {}}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "global FIXATION_CORRECTION_ITERATION \n",
    "FIXATION_CORRECTION_ITERATION[0] = 4\n",
    "print('Fixation correction iteration:', current_fixation_correction_iteration())\n",
    "print('Previous Fixation correction iteration:', previous_fixation_correction_iteration())\n",
    "x_offsets = get_x_offsets()\n",
    "x_offsets"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Include manual evaluation\n",
    "* check manual accuracy evaluation file\n",
    "* exclude unusable trials\n",
    "* copy selected fixations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1b2027253e214714bafcd8447aebea98",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/24 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 003\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n",
      "----------------------------------------------\n",
      "Participant 004\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n",
      "----------------------------------------------\n",
      "Participant 005\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n",
      "----------------------------------------------\n",
      "Participant 007\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n",
      "----------------------------------------------\n",
      "Participant 008\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n",
      "----------------------------------------------\n",
      "Participant 009\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n",
      "----------------------------------------------\n",
      "Participant 010\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n",
      "----------------------------------------------\n",
      "Participant 011\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n",
      "----------------------------------------------\n",
      "Participant 012\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n",
      "----------------------------------------------\n",
      "Participant 013\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n",
      "----------------------------------------------\n",
      "Participant 014\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n",
      "----------------------------------------------\n",
      "Participant 015\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n",
      "----------------------------------------------\n",
      "Participant 016\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n",
      "----------------------------------------------\n",
      "Participant 017\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n",
      "----------------------------------------------\n",
      "Participant 018\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n",
      "----------------------------------------------\n",
      "Participant 019\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n",
      "----------------------------------------------\n",
      "Participant 020\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n",
      "----------------------------------------------\n",
      "Participant 021\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n",
      "----------------------------------------------\n",
      "Participant 022\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n",
      "----------------------------------------------\n",
      "Participant 023\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n",
      "----------------------------------------------\n",
      "Participant 024\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n",
      "----------------------------------------------\n",
      "Participant 101\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n",
      "----------------------------------------------\n",
      "Participant 102\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n",
      "----------------------------------------------\n",
      "Participant 106\n",
      "(01/03) Check manual evaluation file\n",
      "(02/03) Exclude trials without suitable corrections (excluded)\n",
      "(03/03) Copy chosen fixations\n"
     ]
    }
   ],
   "source": [
    "all_usable = True\n",
    "participant_exclusions = {}\n",
    "for participant in tqdm(folders):\n",
    "\n",
    "    print('----------------------------------------------')\n",
    "    print(PARTICIPANT, participant)\n",
    "    manual_accuracy_evaluation_path = get_manual_accuracy_evaluation_path(\n",
    "        participant, performed=True)\n",
    "    if not manual_accuracy_evaluation_path.exists():\n",
    "        print('Performed evaluation path not found:',\n",
    "              manual_accuracy_evaluation_path.as_posix())\n",
    "        continue\n",
    "    print('(01/03) Check manual evaluation file')\n",
    "    manual_evaluation = pd.read_csv(\n",
    "        manual_accuracy_evaluation_path, sep=SEPARATOR, index_col=False, dtype={PARTICIPANT: str})\n",
    "    usable = check_manual_accuracy_evaluation(\n",
    "        manual_evaluation, participant, snippets[participant])\n",
    "    if not usable:\n",
    "        print('\\n--The fixations for this participant have to be re-evaluated in another iteration.')\n",
    "        continue\n",
    "\n",
    "    print('(02/03) Exclude trials without suitable corrections (excluded)')\n",
    "    to_be_excluded = {row[SNIPPET]: row[MANUAL_ACCURACY_EVALUATION_REASON] for _, row in manual_evaluation.iterrows(\n",
    "    ) if row[MANUAL_ACCURACY_EVALUATION_CHOICE] == FIXATION_CORRECTION_ALGORITHM_SNIPPET_EXCLUDED}\n",
    "    participant_exclusions[participant] = to_be_excluded\n",
    "    print('(03/03) Copy chosen fixations')\n",
    "    for _, row in manual_evaluation.iterrows():\n",
    "        algo = row[MANUAL_ACCURACY_EVALUATION_CHOICE]\n",
    "        final_iteration = row[MANUAL_ACCURACY_EVALUATION_ITERATION]\n",
    "        if algo == FIXATION_CORRECTION_ALGORITHM_SNIPPET_EXCLUDED:\n",
    "            continue\n",
    "        snippet_fixations_path = get_fixations_path(participant, row[SNIPPET], fixation_cross=False, modified_version=algo, no_outliers_if_exists=(\n",
    "            algo == FIXATION_CORRECTION_ALGORITHM_ORIGINAL), iteration=lambda:final_iteration)\n",
    "        fixation_data = pd.read_csv(\n",
    "            snippet_fixations_path, index_col=False, sep=SEPARATOR, dtype={PARTICIPANT: str})\n",
    "        if fixation_data.shape[0]<MIN_FIXATION_COUNT_TRIAL:\n",
    "            print(f'{row[SNIPPET]}: trial with less than {MIN_FIXATION_COUNT_TRIAL} fixations, thus excluded')\n",
    "            to_be_excluded[row[SNIPPET]]='Too few fixations'\n",
    "            continue\n",
    "        if participant in x_offsets:\n",
    "            x_offset = get_snippet_x_offset(x_offsets[participant], row[SNIPPET])\n",
    "            if x_offset:\n",
    "                fixation_data[FIXATION_COLUMN_X] = fixation_data[FIXATION_COLUMN_X]+x_offset\n",
    "        fixation_data.to_csv(get_selected_fixations_path(\n",
    "            participant, row[SNIPPET]), sep=SEPARATOR, index=False)\n",
    "    if to_be_excluded:\n",
    "        pass\n",
    "        update_exclusions(participant, SNIPPET, VISUAL,\n",
    "                          {snippet: f'01d7_PRE_VIS_Gaze_Selection_Fixation-it{current_fixation_correction_iteration()}: No fixation correction selected due to unsuitable accuracy. {reason}' for snippet, reason in to_be_excluded.items()})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "exclusion no by participant {'003': 19, '004': 28, '005': 11, '007': 7, '008': 16, '009': 4, '010': 12, '011': 6, '012': 6, '013': 5, '014': 3, '015': 1, '016': 12, '017': 14, '018': 7, '019': 12, '020': 6, '021': 1, '022': 12, '023': 3, '024': 2, '101': 6, '102': 4, '106': 30}\n",
      "total exclusion no 227\n",
      "average exclusion by participant 9.46\n",
      "exclusion rate 13.14 %\n"
     ]
    }
   ],
   "source": [
    "participant_exclusion_count = {participant:len(participant_exclusions[participant]) for participant in participant_exclusions}\n",
    "print(f'exclusion no by participant { participant_exclusion_count}')\n",
    "print('total exclusion no', sum(participant_exclusion_count.values()))\n",
    "print('average exclusion by participant', round(sum(participant_exclusion_count.values())/len(participant_exclusion_count),2),)\n",
    "print('exclusion rate', round(100*sum(participant_exclusion_count.values())/(72*len(participant_exclusion_count)),2),'%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All fixations were corrected without concerns, you can continue with the pipeline.\n"
     ]
    }
   ],
   "source": [
    "if all_usable:\n",
    "    print('All fixations were corrected without concerns, you can continue with the pipeline.')\n",
    "else:\n",
    "    raise Exception(f'The fixations for at least one participant have to be re-evaluated in another iteration. See above.')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}