09-Task-Evaluate_Data/01d4_PRE_VIS_Fixation_Correction-it2.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Perform Fixation Correction Iteration 2\n",
    "Status: Implemented\n",
    " -- Fully automatized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "\n",
    "import pandas as pd\n",
    "from tqdm.notebook import tqdm\n",
    "from utils.utils import *\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    }
   },
   "outputs": [],
   "source": [
    "# get all files from participants\n",
    "folders = get_participant_folder_per_participant()\n",
    "sequence_orders = get_sequence_order_per_participant()\n",
    "assert (set(folders.keys()) == set(sequence_orders.keys()))\n",
    "snippets = {p: list(sequence_orders[p][SNIPPET].values)\n",
    "            for p in sequence_orders}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fixation correction iteration: 2\n",
      "Previous Fixation correction iteration: 1\n"
     ]
    }
   ],
   "source": [
    "global FIXATION_CORRECTION_ITERATION \n",
    "FIXATION_CORRECTION_ITERATION[0] = 2\n",
    "print('Fixation correction iteration:', current_fixation_correction_iteration())\n",
    "print('Previous Fixation correction iteration:', previous_fixation_correction_iteration())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Chars</th>\n",
       "      <th>EssentialChars</th>\n",
       "      <th>LoC</th>\n",
       "      <th>EssentialLoC</th>\n",
       "      <th>Max length LoC</th>\n",
       "      <th>Words</th>\n",
       "      <th>EssentialWords</th>\n",
       "      <th>CodeLines</th>\n",
       "      <th>EssentialCodeLines</th>\n",
       "      <th>number</th>\n",
       "      <th>version</th>\n",
       "      <th>Snippet_base</th>\n",
       "      <th>variant</th>\n",
       "      <th>Condition</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Snippet</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>-clean-v0</th>\n",
       "      <td>49</td>\n",
       "      <td>33</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>26</td>\n",
       "      <td>16</td>\n",
       "      <td>10</td>\n",
       "      <td>['int V1 = 10;', 'int V2 = 3;', 'boolean R = (...</td>\n",
       "      <td>[1, 1, 1]</td>\n",
       "      <td>0</td>\n",
       "      <td>v0</td>\n",
       "      <td>--v0</td>\n",
       "      <td>clean</td>\n",
       "      <td>clean</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-clean-v1</th>\n",
       "      <td>48</td>\n",
       "      <td>32</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>26</td>\n",
       "      <td>16</td>\n",
       "      <td>10</td>\n",
       "      <td>['int V1 = 4;', 'int V2 = 2;', 'boolean R = (V...</td>\n",
       "      <td>[1, 1, 1]</td>\n",
       "      <td>0</td>\n",
       "      <td>v1</td>\n",
       "      <td>--v1</td>\n",
       "      <td>clean</td>\n",
       "      <td>clean</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-clean-v2</th>\n",
       "      <td>48</td>\n",
       "      <td>32</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>26</td>\n",
       "      <td>16</td>\n",
       "      <td>10</td>\n",
       "      <td>['int V1 = 1;', 'int V2 = 5;', 'boolean R = (V...</td>\n",
       "      <td>[1, 1, 1]</td>\n",
       "      <td>0</td>\n",
       "      <td>v2</td>\n",
       "      <td>--v2</td>\n",
       "      <td>clean</td>\n",
       "      <td>clean</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-obf-v0</th>\n",
       "      <td>54</td>\n",
       "      <td>36</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>31</td>\n",
       "      <td>18</td>\n",
       "      <td>10</td>\n",
       "      <td>['int V1 = 10;', 'int V2 = 3;', 'boolean R = (...</td>\n",
       "      <td>[1, 1, 1]</td>\n",
       "      <td>0</td>\n",
       "      <td>v0</td>\n",
       "      <td>--v0</td>\n",
       "      <td>obf</td>\n",
       "      <td>confusing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>-obf-v1</th>\n",
       "      <td>54</td>\n",
       "      <td>36</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>32</td>\n",
       "      <td>18</td>\n",
       "      <td>10</td>\n",
       "      <td>['int V1 = 4;', 'int V2 = 2;', 'boolean R = ( ...</td>\n",
       "      <td>[1, 1, 1]</td>\n",
       "      <td>0</td>\n",
       "      <td>v1</td>\n",
       "      <td>--v1</td>\n",
       "      <td>obf</td>\n",
       "      <td>confusing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9-clean-v1</th>\n",
       "      <td>37</td>\n",
       "      <td>26</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>21</td>\n",
       "      <td>11</td>\n",
       "      <td>7</td>\n",
       "      <td>['int V1 = 5;', 'boolean R = (V1 &gt; 2);', 'V1--;']</td>\n",
       "      <td>[1, 1, 1]</td>\n",
       "      <td>9</td>\n",
       "      <td>v1</td>\n",
       "      <td>9--v1</td>\n",
       "      <td>clean</td>\n",
       "      <td>clean</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9-clean-v2</th>\n",
       "      <td>38</td>\n",
       "      <td>27</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>22</td>\n",
       "      <td>11</td>\n",
       "      <td>7</td>\n",
       "      <td>['int V1 = 8;', 'boolean R = (V1 &lt;= 7);', 'V1-...</td>\n",
       "      <td>[1, 1, 1]</td>\n",
       "      <td>9</td>\n",
       "      <td>v2</td>\n",
       "      <td>9--v2</td>\n",
       "      <td>clean</td>\n",
       "      <td>clean</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9-obf-v0</th>\n",
       "      <td>35</td>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>24</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>['int V1 = 2;', 'boolean R = (V1-- == 1);']</td>\n",
       "      <td>[1, 1]</td>\n",
       "      <td>9</td>\n",
       "      <td>v0</td>\n",
       "      <td>9--v0</td>\n",
       "      <td>obf</td>\n",
       "      <td>confusing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9-obf-v1</th>\n",
       "      <td>34</td>\n",
       "      <td>24</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>23</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>['int V1 = 5;', 'boolean R = (V1-- &gt; 2);']</td>\n",
       "      <td>[1, 1]</td>\n",
       "      <td>9</td>\n",
       "      <td>v1</td>\n",
       "      <td>9--v1</td>\n",
       "      <td>obf</td>\n",
       "      <td>confusing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9-obf-v2</th>\n",
       "      <td>35</td>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>24</td>\n",
       "      <td>10</td>\n",
       "      <td>6</td>\n",
       "      <td>['int V1 = 8;', 'boolean R = (V1-- &lt;= 7);']</td>\n",
       "      <td>[1, 1]</td>\n",
       "      <td>9</td>\n",
       "      <td>v2</td>\n",
       "      <td>9--v2</td>\n",
       "      <td>obf</td>\n",
       "      <td>confusing</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>144 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Chars  EssentialChars  LoC  EssentialLoC  Max length LoC  Words   \n",
       "Snippet                                                                       \n",
       "-clean-v0      49              33    3             3              26     16  \\\n",
       "-clean-v1      48              32    3             3              26     16   \n",
       "-clean-v2      48              32    3             3              26     16   \n",
       "-obf-v0        54              36    3             3              31     18   \n",
       "-obf-v1        54              36    3             3              32     18   \n",
       "...           ...             ...  ...           ...             ...    ...   \n",
       "9-clean-v1     37              26    3             3              21     11   \n",
       "9-clean-v2     38              27    3             3              22     11   \n",
       "9-obf-v0       35              25    2             2              24     10   \n",
       "9-obf-v1       34              24    2             2              23     10   \n",
       "9-obf-v2       35              25    2             2              24     10   \n",
       "\n",
       "            EssentialWords                                          CodeLines   \n",
       "Snippet                                                                         \n",
       "-clean-v0               10  ['int V1 = 10;', 'int V2 = 3;', 'boolean R = (...  \\\n",
       "-clean-v1               10  ['int V1 = 4;', 'int V2 = 2;', 'boolean R = (V...   \n",
       "-clean-v2               10  ['int V1 = 1;', 'int V2 = 5;', 'boolean R = (V...   \n",
       "-obf-v0                 10  ['int V1 = 10;', 'int V2 = 3;', 'boolean R = (...   \n",
       "-obf-v1                 10  ['int V1 = 4;', 'int V2 = 2;', 'boolean R = ( ...   \n",
       "...                    ...                                                ...   \n",
       "9-clean-v1               7  ['int V1 = 5;', 'boolean R = (V1 > 2);', 'V1--;']   \n",
       "9-clean-v2               7  ['int V1 = 8;', 'boolean R = (V1 <= 7);', 'V1-...   \n",
       "9-obf-v0                 6        ['int V1 = 2;', 'boolean R = (V1-- == 1);']   \n",
       "9-obf-v1                 6         ['int V1 = 5;', 'boolean R = (V1-- > 2);']   \n",
       "9-obf-v2                 6        ['int V1 = 8;', 'boolean R = (V1-- <= 7);']   \n",
       "\n",
       "           EssentialCodeLines  number version Snippet_base variant  Condition  \n",
       "Snippet                                                                        \n",
       "-clean-v0           [1, 1, 1]       0      v0         --v0   clean      clean  \n",
       "-clean-v1           [1, 1, 1]       0      v1         --v1   clean      clean  \n",
       "-clean-v2           [1, 1, 1]       0      v2         --v2   clean      clean  \n",
       "-obf-v0             [1, 1, 1]       0      v0         --v0     obf  confusing  \n",
       "-obf-v1             [1, 1, 1]       0      v1         --v1     obf  confusing  \n",
       "...                       ...     ...     ...          ...     ...        ...  \n",
       "9-clean-v1          [1, 1, 1]       9      v1        9--v1   clean      clean  \n",
       "9-clean-v2          [1, 1, 1]       9      v2        9--v2   clean      clean  \n",
       "9-obf-v0               [1, 1]       9      v0        9--v0     obf  confusing  \n",
       "9-obf-v1               [1, 1]       9      v1        9--v1     obf  confusing  \n",
       "9-obf-v2               [1, 1]       9      v2        9--v2     obf  confusing  \n",
       "\n",
       "[144 rows x 14 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "code_sizes = pd.read_csv(CODE_SIZE_PATH, sep=SEPARATOR,\n",
    "                         index_col=False).set_index(SNIPPET)\n",
    "display(code_sizes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Take into account previous iteration of manual evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There exist manual accuracy evaluation files from the previous iteration which will be considered for this iteration's fixation correction. \n",
      "{'003': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/003_DataAccuracy_manual_evaluation_combined.csv'), '004': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/004_DataAccuracy_manual_evaluation_combined.csv'), '005': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/005_DataAccuracy_manual_evaluation_combined.csv'), '007': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/007_DataAccuracy_manual_evaluation_combined.csv'), '008': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/008_DataAccuracy_manual_evaluation_combined.csv'), '009': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/009_DataAccuracy_manual_evaluation_combined.csv'), '010': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/010_DataAccuracy_manual_evaluation_combined.csv'), '011': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/011_DataAccuracy_manual_evaluation_combined.csv'), '012': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/012_DataAccuracy_manual_evaluation_combined.csv'), '013': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/013_DataAccuracy_manual_evaluation_combined.csv'), '014': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/014_DataAccuracy_manual_evaluation_combined.csv'), '015': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/015_DataAccuracy_manual_evaluation_combined.csv'), '016': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/016_DataAccuracy_manual_evaluation_combined.csv'), '017': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/017_DataAccuracy_manual_evaluation_combined.csv'), '018': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/018_DataAccuracy_manual_evaluation_combined.csv'), '019': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/019_DataAccuracy_manual_evaluation_combined.csv'), '020': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/020_DataAccuracy_manual_evaluation_combined.csv'), '021': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/021_DataAccuracy_manual_evaluation_combined.csv'), '022': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/022_DataAccuracy_manual_evaluation_combined.csv'), '023': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/023_DataAccuracy_manual_evaluation_combined.csv'), '024': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/024_DataAccuracy_manual_evaluation_combined.csv'), '101': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/101_DataAccuracy_manual_evaluation_combined.csv'), '102': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/102_DataAccuracy_manual_evaluation_combined.csv'), '106': WindowsPath('../08-Data-Trial_Recordings/manual_accuracy_evaluation/iteration_1/combined/106_DataAccuracy_manual_evaluation_combined.csv')}\n",
      "----------------------------------------------\n",
      "Participant 003\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 004\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 005\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 007\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 008\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 009\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 010\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 011\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 012\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 013\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 014\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 015\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 016\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 017\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 018\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 019\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 020\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 021\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 022\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 023\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 024\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 101\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 102\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n",
      "----------------------------------------------\n",
      "Participant 106\n",
      "(01/04) Load accuracy evaluation file\n",
      "(02/04) Select those in need of rework\n",
      "(03/04) Check that outliers were removed correctly and offsets were adapted\n",
      "(04/04) Save snippets to rework\n"
     ]
    }
   ],
   "source": [
    "# check whether there already exist manual evaluation files\n",
    "manual_evaluation_paths = exist_previous_manual_accuracy_evaluation_paths()\n",
    "if manual_evaluation_paths and current_fixation_correction_iteration() > 1:\n",
    "    assert (set(folders.keys()) == set(manual_evaluation_paths.keys())), f'''folder: {folders.keys()}, \\neval paths: {manual_evaluation_paths.keys()}'''\n",
    "\n",
    "    print('There exist manual accuracy evaluation files from the previous iteration which will be considered for this iteration\\'s fixation correction. ')\n",
    "    print(manual_evaluation_paths)\n",
    "    for participant in manual_evaluation_paths:\n",
    "        print('----------------------------------------------')\n",
    "        print(PARTICIPANT, participant)\n",
    "\n",
    "        print('(01/04) Load accuracy evaluation file')\n",
    "        manual_evaluation = pd.read_csv(\n",
    "            manual_evaluation_paths[participant], sep=SEPARATOR, index_col=False, dtype={PARTICIPANT: str})\n",
    "\n",
    "        print('(02/04) Select those in need of rework')\n",
    "        to_rework = get_to_rework_manual_accuracy_evaluation(\n",
    "            manual_evaluation, participant, snippets[participant])\n",
    "\n",
    "        print('(03/04) Check that outliers were removed correctly and offsets were adapted')\n",
    "        check_outlier_offset_actions(to_rework, participant)\n",
    "\n",
    "        print('(04/04) Save snippets to rework')\n",
    "        # save snippet names to rework in file\n",
    "        to_rework[SNIPPET].to_csv(get_snippet_rework_paths(\n",
    "            participant), sep=SEPARATOR, index=False)\n",
    "        snippets[participant] = to_rework[SNIPPET].to_list()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Auto-correction\n",
    "* perform auto-correction with various algorithms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd62553564aa471fbb4b26f3b9b26b8f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/24 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 003\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4511930e87294cc4bafe1d02a36e66d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/67 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "95549b9de58f4c2c871377c43c2f1a29",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/67 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f85064a86d3d4a35b625f0fc2833f103",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/67 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 004\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5e95d4ca4e6a4b67915613756ee409bc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/57 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7967a51ebe0d40cba4c1968e4d5e52ab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/57 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a6595eb1650d4011be1679fe90d3bbd6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/57 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 005\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "08571003096a4b5188883bb324bd2e2f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/71 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a7cff01f3d3410d8adf029ac7a5a772",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/71 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7346db59085496a94482916c5e9935f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/71 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 007\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fdfba60de0424b548f78b4f8d5adc5cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4552162c78594cfabc41ad64b2622849",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2fc79105ee1145f3903288a61d353891",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 008\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8180a7b7c2dd48e8acc2a5fefa39ce13",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/68 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e5675d9014f54827ae160e786d5045a9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/68 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "377c9087a1ba4da7835dfe659dc6e119",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/68 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 009\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "401b060c06ff452b8d16c0d80cad9a89",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f1bcbba4d8e948729f0c14e0a6d66dee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1ab00839ccef4ea29883b9aaac1312fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 010\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33fb7fb8049645b9b3e179b003e7dae3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/67 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8f0dad9199e4439e8f222fa6a2f7371e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/67 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "709050b8a066442cb7f451af552bbcef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/67 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 011\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af5e78e174ff4fde83959b2104804728",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/71 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ebd574659fa9486f85500be6d3d9264a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/71 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d9be1bc5e9d946d98581a9df8b564a95",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/71 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 012\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3d811ce502b4355b9cca66eea1b7a82",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/71 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d0076fc4fd04298aba1a33f47bba140",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/71 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "737458ee0ccb463897c9a26b6779eb2f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/71 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 013\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3aa872f6d76e4ad6b99d06ad6797a1b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/68 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9a33bf11b5ce4d58989e56aa2aa23874",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/68 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e4615385598412992acd9a59a3daaac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/68 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 014\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e9556d7f2ebb4a3dafac21f9a6dedc24",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/70 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2e8eaa957a4a4a08bc3e48d311ca552a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/70 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "93485c77ed8e4c9fa787fe47a83dfb1f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/70 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 015\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "379cae3af9c248ae90c3560514e1459c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d1942bf177ed49bb8999d453030b3813",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f557aa5db6b4e77a7348f347a4c93c9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 016\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d4c671d57de14a3788a57ea0ae15d003",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "11fe118cf1e942f0b66b65c1c48699c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9deb05f9903645fa944ce2555d35df1a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 017\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4130f6806ef540daa03a2eac5fd93297",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/68 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9fa2817f37fc4e9594c175738b7de082",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/68 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4f074f0e1e1d40c086914a0446feae32",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/68 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 018\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fda3815dbda84b31b6587de410f5ab73",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/68 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e8fbaa50e1a649309436d5d3c25bb278",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/68 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "38e7c964ec964c119f6da6054b25f2b0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/68 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 019\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a68dfd81d624b349a050b9a028dad97",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/66 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "30b6cf3c0a0944b7bf6e8e7c2a34033e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/66 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33c2b7027a2949e8aadec1d93db5af89",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/66 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 020\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4196a0c4cf634a43ab6bd783062480d1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/71 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3bdede84b6294be2a1f94978638dcaa9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/71 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e529ebb66bee487aab1570fca53af9fa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/71 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 021\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "391d174b129c4def92fe7df9d2002b92",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "789d98c85cdf41c59f4c167dc5905e93",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5d52c2b31a274e4c9eaebb65d3f3bb36",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 022\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0d16a4b2d18345c8a9e320140efb1661",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/69 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c427b6071c324e679b035d7beb0128e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/69 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "\tNo corrected fixation data possible for 49-obf-v0 for cluster_alllines\n",
      "\tNo corrected fixation data possible for 63-clean-v0 for cluster_alllines\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d0f486c5099f43e18e49ebece5d07221",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/69 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 023\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0c37b9b9f5c42b6a327656dec90c7b0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/71 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1858412376c4b8692272d94a0e0649d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/71 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cacde3e6fe9c4c98b4a1b1f3466b1a4b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/71 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 024\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "313a22113fce444e998ec3a814e43c91",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8c9a433fe55243fa8ad2acf57e73148d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "98a21316b479433a8f469410a3d7653a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 101\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b48b3c8c55346689acf6903a4c39231",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "adcab11c29714152a93c33987b1ba789",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa76a38672ba4b07a57d448bbe126a58",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 102\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "73e75ca699684cd8bacfd80521333202",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/71 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a436abd13756427db9f233f33e3095ab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/71 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1555a572d06f46b1b317d80bc1cc681e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/71 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------\n",
      "Participant 106\n",
      "(01/04) Delete existing fixation corrections\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fd3df7c2ba5944a48d59dae3c9b9581b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(02/04) Retrieve fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "58c7d2dd2d0a4180acd61050c3bd5099",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(03/04) Perform auto-correction\n",
      "(04/04) Store corrected fixation data\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ab0d960f52040b49ec191ac1d17ec7b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/72 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if current_fixation_correction_iteration() > 1:\n",
    "    # Iterate over all participants\n",
    "    for participant in tqdm(folders):\n",
    "        print('----------------------------------------------')\n",
    "        print(PARTICIPANT, participant)\n",
    "        if not snippets[participant]:\n",
    "            print('no snippets to (re-)work found.')\n",
    "            continue\n",
    "\n",
    "        print('(01/04) Delete existing fixation corrections')\n",
    "        for snippet in tqdm(snippets[participant]):\n",
    "            fixation_correction_paths = get_all_fixation_correction_paths(\n",
    "                participant, snippet)\n",
    "            for p in fixation_correction_paths:\n",
    "                p.unlink()\n",
    "\n",
    "        print('(02/04) Retrieve fixation data')\n",
    "        participant_snippet_fixations = {}\n",
    "        not_available = set()\n",
    "        for snippet in tqdm(snippets[participant]):\n",
    "            snippet_fixations_path = get_fixations_path(\n",
    "                participant, snippet, no_outliers_if_exists=True, iteration=previous_fixation_correction_iteration)\n",
    "            if snippet_fixations_path.exists():\n",
    "                participant_snippet_fixations[snippet] = pd.read_csv(\n",
    "                    snippet_fixations_path, index_col=False, sep=SEPARATOR, dtype={PARTICIPANT: str})\n",
    "                participant_snippet_fixations[snippet].to_csv(get_fixations_path(\n",
    "                    participant, snippet, no_outliers_if_exists=False), index=False, sep=SEPARATOR)\n",
    "            else:\n",
    "                not_available.add(snippet)\n",
    "        if not_available:\n",
    "            update_exclusions(participant, SNIPPET, VISUAL, {snippet: f'''01d4_PRE_VIS_Fixation_Correction-it{\n",
    "                              current_fixation_correction_iteration()}: Fixation data not available''' for snippet in not_available})\n",
    "        for snippet in not_available:\n",
    "            snippets[participant].remove(snippet)\n",
    "\n",
    "        print('(03/04) Perform auto-correction')\n",
    "        corrected_snippet_fixations = calculate_corrected_fixations(\n",
    "            participant_snippet_fixations, code_sizes)\n",
    "\n",
    "        print('(04/04) Store corrected fixation data')\n",
    "        # save fixation data\n",
    "        for snippet in tqdm(corrected_snippet_fixations):\n",
    "            corrected_fixation_datasets = corrected_snippet_fixations[snippet]\n",
    "            for key, fixation_data in corrected_fixation_datasets.items():\n",
    "                if key == FIXATION_CORRECTION_ALGORITHM_ORIGINAL:\n",
    "                    continue\n",
    "                fixation_data.to_csv(get_fixations_path(\n",
    "                    participant, snippet, modified_version=key), index=False, sep=SEPARATOR)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}