nb_dataset_create_alt_model_training_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook generates the training data for all models (PCA, KMeans, Autoencoder) used during analysis"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# All Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "task_name = 'alt_all_data'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import warnings\n",
    "from tqdm import tqdm_notebook as tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Num Files: 223\n"
     ]
    }
   ],
   "source": [
    "path = './data/alt_openface_old_protocol'\n",
    "tasks = ['Task1','Task2','Task3']\n",
    "\n",
    "files = []\n",
    "for task in tasks:\n",
    "    subpath = os.path.join(path,task)\n",
    "    files += [os.path.join(task,f) for f in os.listdir(subpath) if os.path.isfile(os.path.join(subpath,f))]\n",
    "\n",
    "print('Num Files:',len(files))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/curt/analysisEnv/lib/python3.6/site-packages/ipykernel_launcher.py:10: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
      "  # Remove the CWD from sys.path while we load stuff.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b44a5faa8cf4b51b8b6a110a9d76178",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/223 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task1/01_0_1.csv\n",
      "Full: (4396, 138)\n",
      "Train: (2945, 138) Test: (725, 138) Validation (726, 138)\n",
      "Task1/02_0_1.csv\n",
      "Full: (4404, 138)\n",
      "Train: (2950, 138) Test: (727, 138) Validation (727, 138)\n",
      "Task1/03_1_1.csv\n",
      "Full: (5314, 138)\n",
      "Train: (3560, 138) Test: (877, 138) Validation (877, 138)\n",
      "Task1/04_1_1.csv\n",
      "Full: (5391, 138)\n",
      "Train: (3611, 138) Test: (890, 138) Validation (890, 138)\n",
      "Task1/05_1_1.csv\n",
      "Full: (8967, 138)\n",
      "Train: (6007, 138) Test: (1480, 138) Validation (1480, 138)\n",
      "Task1/07_0_1.csv\n",
      "Full: (8283, 138)\n",
      "Train: (5549, 138) Test: (1367, 138) Validation (1367, 138)\n",
      "Task1/08_0_1.csv\n",
      "Full: (7886, 138)\n",
      "Train: (5283, 138) Test: (1301, 138) Validation (1302, 138)\n",
      "Task1/09_0_1.csv\n",
      "Full: (6800, 138)\n",
      "Train: (4556, 138) Test: (1122, 138) Validation (1122, 138)\n",
      "Task1/100_0_1.csv\n",
      "Full: (4289, 138)\n",
      "Train: (2873, 138) Test: (708, 138) Validation (708, 138)\n",
      "Task1/101_0_1.csv\n",
      "Full: (4370, 138)\n",
      "Train: (2927, 138) Test: (721, 138) Validation (722, 138)\n",
      "Task1/102_0_1.csv\n",
      "Full: (4258, 138)\n",
      "Train: (2852, 138) Test: (703, 138) Validation (703, 138)\n",
      "Task1/103_1_1.csv\n",
      "Full: (4852, 138)\n",
      "Train: (3250, 138) Test: (801, 138) Validation (801, 138)\n",
      "Task1/104_1_1.csv\n",
      "Full: (4838, 138)\n",
      "Train: (3241, 138) Test: (798, 138) Validation (799, 138)\n",
      "Task1/105_0_1.csv\n",
      "Full: (6043, 138)\n",
      "Train: (4048, 138) Test: (997, 138) Validation (998, 138)\n",
      "Task1/106_0_1.csv\n",
      "Full: (5953, 138)\n",
      "Train: (3988, 138) Test: (982, 138) Validation (983, 138)\n",
      "Task1/107_1_1.csv\n",
      "Full: (6958, 138)\n",
      "Train: (4661, 138) Test: (1148, 138) Validation (1149, 138)\n",
      "Task1/108_1_1.csv\n",
      "Full: (6891, 138)\n",
      "Train: (4616, 138) Test: (1137, 138) Validation (1138, 138)\n",
      "Task1/109_1_1.csv\n",
      "Full: (7320, 138)\n",
      "Train: (4904, 138) Test: (1208, 138) Validation (1208, 138)\n",
      "Task1/10_0_1.csv\n",
      "Full: (6860, 138)\n",
      "Train: (4596, 138) Test: (1132, 138) Validation (1132, 138)\n",
      "Task1/11_0_1.csv\n",
      "Full: (5464, 138)\n",
      "Train: (3660, 138) Test: (902, 138) Validation (902, 138)\n",
      "Task1/12_0_1.csv\n",
      "Full: (5425, 138)\n",
      "Train: (3634, 138) Test: (895, 138) Validation (896, 138)\n",
      "Task1/13_1_1.csv\n",
      "Full: (6885, 138)\n",
      "Train: (4612, 138) Test: (1136, 138) Validation (1137, 138)\n",
      "Task1/14_1_1.csv\n",
      "Full: (6556, 138)\n",
      "Train: (4392, 138) Test: (1082, 138) Validation (1082, 138)\n",
      "Task1/15_0_1.csv\n",
      "Full: (8452, 138)\n",
      "Train: (5662, 138) Test: (1395, 138) Validation (1395, 138)\n",
      "Task1/16_0_1.csv\n",
      "Full: (5419, 138)\n",
      "Train: (3630, 138) Test: (894, 138) Validation (895, 138)\n",
      "Task1/17_1_1.csv\n",
      "Full: (6387, 138)\n",
      "Train: (4279, 138) Test: (1054, 138) Validation (1054, 138)\n",
      "Task1/18_1_1.csv\n",
      "Full: (6371, 138)\n",
      "Train: (4268, 138) Test: (1051, 138) Validation (1052, 138)\n",
      "Task1/25_1_1.csv\n",
      "Full: (4725, 138)\n",
      "Train: (3165, 138) Test: (780, 138) Validation (780, 138)\n",
      "Task1/26_1_1.csv\n",
      "Full: (7997, 138)\n",
      "Train: (5357, 138) Test: (1320, 138) Validation (1320, 138)\n",
      "Task1/27_0_1.csv\n",
      "Full: (4065, 138)\n",
      "Train: (2723, 138) Test: (671, 138) Validation (671, 138)\n",
      "Task1/28_0_1.csv\n",
      "Full: (5404, 138)\n",
      "Train: (3620, 138) Test: (892, 138) Validation (892, 138)\n",
      "Task1/29_1_1.csv\n",
      "Full: (4671, 138)\n",
      "Train: (3129, 138) Test: (771, 138) Validation (771, 138)\n",
      "Task1/30_1_1.csv\n",
      "Full: (4839, 138)\n",
      "Train: (3242, 138) Test: (798, 138) Validation (799, 138)\n",
      "Task1/31_1_1.csv\n",
      "Full: (9838, 138)\n",
      "Train: (6591, 138) Test: (1623, 138) Validation (1624, 138)\n",
      "Task1/32_1_1.csv\n",
      "Full: (9811, 138)\n",
      "Train: (6573, 138) Test: (1619, 138) Validation (1619, 138)\n",
      "Task1/35_1_1.csv\n",
      "Full: (5747, 138)\n",
      "Train: (3850, 138) Test: (948, 138) Validation (949, 138)\n",
      "Task1/36_1_1.csv\n",
      "Full: (5790, 138)\n",
      "Train: (3879, 138) Test: (955, 138) Validation (956, 138)\n",
      "Task1/37_0_1.csv\n",
      "Full: (4174, 138)\n",
      "Train: (2796, 138) Test: (689, 138) Validation (689, 138)\n",
      "Task1/38_0_1.csv\n",
      "Full: (4159, 138)\n",
      "Train: (2786, 138) Test: (686, 138) Validation (687, 138)\n",
      "Task1/41_0_1.csv\n",
      "Full: (11164, 138)\n",
      "Train: (7479, 138) Test: (1842, 138) Validation (1843, 138)\n",
      "Task1/42_0_1.csv\n",
      "Full: (11036, 138)\n",
      "Train: (7394, 138) Test: (1821, 138) Validation (1821, 138)\n",
      "Task1/43_1_1.csv\n",
      "Full: (6175, 138)\n",
      "Train: (4137, 138) Test: (1019, 138) Validation (1019, 138)\n",
      "Task1/44_1_1.csv\n",
      "Full: (6338, 138)\n",
      "Train: (4246, 138) Test: (1046, 138) Validation (1046, 138)\n",
      "Task1/45_1_1.csv\n",
      "Full: (6259, 138)\n",
      "Train: (4193, 138) Test: (1033, 138) Validation (1033, 138)\n",
      "Task1/46_1_1.csv\n",
      "Full: (6286, 138)\n",
      "Train: (4211, 138) Test: (1037, 138) Validation (1038, 138)\n",
      "Task1/47_1_1.csv\n",
      "Full: (5338, 138)\n",
      "Train: (3576, 138) Test: (881, 138) Validation (881, 138)\n",
      "Task1/48_1_1.csv\n",
      "Full: (5724, 138)\n",
      "Train: (3835, 138) Test: (944, 138) Validation (945, 138)\n",
      "Task1/49_1_1.csv\n",
      "Full: (8036, 138)\n",
      "Train: (5384, 138) Test: (1326, 138) Validation (1326, 138)\n",
      "Task1/50_1_1.csv\n",
      "Full: (7928, 138)\n",
      "Train: (5311, 138) Test: (1308, 138) Validation (1309, 138)\n",
      "Task1/51_1_1-p1.csv\n",
      "Full: (4249, 138)\n",
      "Train: (2846, 138) Test: (701, 138) Validation (702, 138)\n",
      "Task1/51_1_1-p2.csv\n",
      "Full: (1790, 138)\n",
      "Train: (1199, 138) Test: (295, 138) Validation (296, 138)\n",
      "Task1/52_1_1.csv\n",
      "Full: (6730, 138)\n",
      "Train: (4509, 138) Test: (1110, 138) Validation (1111, 138)\n",
      "Task1/55_0_1.csv\n",
      "Full: (13199, 138)\n",
      "Train: (8843, 138) Test: (2178, 138) Validation (2178, 138)\n",
      "Task1/56_0_1.csv\n",
      "Full: (13215, 138)\n",
      "Train: (8854, 138) Test: (2180, 138) Validation (2181, 138)\n",
      "Task1/59_1_1.csv\n",
      "Full: (5237, 138)\n",
      "Train: (3508, 138) Test: (864, 138) Validation (865, 138)\n",
      "Task1/60_1_1.csv\n",
      "Full: (5237, 138)\n",
      "Train: (3508, 138) Test: (864, 138) Validation (865, 138)\n",
      "Task1/62_1_1.csv\n",
      "Full: (3441, 138)\n",
      "Train: (2305, 138) Test: (568, 138) Validation (568, 138)\n",
      "Task1/63_0_1.csv\n",
      "Full: (11408, 138)\n",
      "Train: (7643, 138) Test: (1882, 138) Validation (1883, 138)\n",
      "Task1/64_0_1.csv\n",
      "Full: (11417, 138)\n",
      "Train: (7649, 138) Test: (1884, 138) Validation (1884, 138)\n",
      "Task1/65_0_1.csv\n",
      "Full: (5666, 138)\n",
      "Train: (3796, 138) Test: (935, 138) Validation (935, 138)\n",
      "Task1/66_0_1.csv\n",
      "Full: (5634, 138)\n",
      "Train: (3774, 138) Test: (930, 138) Validation (930, 138)\n",
      "Task1/68_1_1.csv\n",
      "Full: (1390, 138)\n",
      "Train: (931, 138) Test: (229, 138) Validation (230, 138)\n",
      "Task1/69_0_1.csv\n",
      "Full: (6671, 138)\n",
      "Train: (4469, 138) Test: (1101, 138) Validation (1101, 138)\n",
      "Task1/70_0_1.csv\n",
      "Full: (6670, 138)\n",
      "Train: (4468, 138) Test: (1101, 138) Validation (1101, 138)\n",
      "Task1/73_1_1.csv\n",
      "Full: (675, 138)\n",
      "Train: (452, 138) Test: (111, 138) Validation (112, 138)\n",
      "Task1/74_1_1.csv\n",
      "Full: (9065, 138)\n",
      "Train: (6073, 138) Test: (1496, 138) Validation (1496, 138)\n",
      "Task1/75_1_1.csv\n",
      "Full: (7277, 138)\n",
      "Train: (4875, 138) Test: (1201, 138) Validation (1201, 138)\n",
      "Task1/76_1_1.csv\n",
      "Full: (7296, 138)\n",
      "Train: (4888, 138) Test: (1204, 138) Validation (1204, 138)\n",
      "Task1/77_0_1.csv\n",
      "Full: (4024, 138)\n",
      "Train: (2696, 138) Test: (664, 138) Validation (664, 138)\n",
      "Task1/78_0_1.csv\n",
      "Full: (3970, 138)\n",
      "Train: (2659, 138) Test: (655, 138) Validation (656, 138)\n",
      "Task1/79_1_1.csv\n",
      "Full: (11290, 138)\n",
      "Train: (7564, 138) Test: (1863, 138) Validation (1863, 138)\n",
      "Task1/80_1_1.csv\n",
      "Full: (11203, 138)\n",
      "Train: (7506, 138) Test: (1848, 138) Validation (1849, 138)\n",
      "Task1/81_1_1.csv\n",
      "Full: (4682, 138)\n",
      "Train: (3136, 138) Test: (773, 138) Validation (773, 138)\n",
      "Task1/82_1_1.csv\n",
      "Full: (4665, 138)\n",
      "Train: (3125, 138) Test: (770, 138) Validation (770, 138)\n",
      "Task1/83_0_1.csv\n",
      "Full: (3412, 138)\n",
      "Train: (2286, 138) Test: (563, 138) Validation (563, 138)\n",
      "Task1/84_0_1.csv\n",
      "Full: (3410, 138)\n",
      "Train: (2284, 138) Test: (563, 138) Validation (563, 138)\n",
      "Task1/87_1_1.csv\n",
      "Full: (4283, 138)\n",
      "Train: (2869, 138) Test: (707, 138) Validation (707, 138)\n",
      "Task1/88_1_1.csv\n",
      "Full: (4283, 138)\n",
      "Train: (2869, 138) Test: (707, 138) Validation (707, 138)\n",
      "Task1/95_1_1.csv\n",
      "Full: (9540, 138)\n",
      "Train: (6391, 138) Test: (1574, 138) Validation (1575, 138)\n",
      "Task1/97_0_1.csv\n",
      "Full: (5635, 138)\n",
      "Train: (3775, 138) Test: (930, 138) Validation (930, 138)\n",
      "Task1/98_0_1.csv\n",
      "Full: (5641, 138)\n",
      "Train: (3779, 138) Test: (931, 138) Validation (931, 138)\n",
      "Task1/99_0_1.csv\n",
      "Full: (4290, 138)\n",
      "Train: (2874, 138) Test: (708, 138) Validation (708, 138)\n",
      "Task2/01_0_2.csv\n",
      "Full: (12810, 138)\n",
      "Train: (8582, 138) Test: (2114, 138) Validation (2114, 138)\n",
      "Task2/02_0_2.csv\n",
      "Full: (12950, 138)\n",
      "Train: (8676, 138) Test: (2137, 138) Validation (2137, 138)\n",
      "Task2/05_1_2.csv\n",
      "Full: (7520, 138)\n",
      "Train: (5038, 138) Test: (1241, 138) Validation (1241, 138)\n",
      "Task2/07_0_2.csv\n",
      "Full: (5085, 138)\n",
      "Train: (3406, 138) Test: (839, 138) Validation (840, 138)\n",
      "Task2/08_0_2.csv\n",
      "Full: (7927, 138)\n",
      "Train: (5311, 138) Test: (1308, 138) Validation (1308, 138)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task2/09_0_2.csv\n",
      "Full: (10915, 138)\n",
      "Train: (7313, 138) Test: (1801, 138) Validation (1801, 138)\n",
      "Task2/101_0_2.csv\n",
      "Full: (5468, 138)\n",
      "Train: (3663, 138) Test: (902, 138) Validation (903, 138)\n",
      "Task2/102_0_2.csv\n",
      "Full: (5478, 138)\n",
      "Train: (3670, 138) Test: (904, 138) Validation (904, 138)\n",
      "Task2/103_1_2.csv\n",
      "Full: (9139, 138)\n",
      "Train: (6123, 138) Test: (1508, 138) Validation (1508, 138)\n",
      "Task2/104_1_2.csv\n",
      "Full: (9102, 138)\n",
      "Train: (6098, 138) Test: (1502, 138) Validation (1502, 138)\n",
      "Task2/105_0_2.csv\n",
      "Full: (17803, 138)\n",
      "Train: (11928, 138) Test: (2937, 138) Validation (2938, 138)\n",
      "Task2/106_0_2.csv\n",
      "Full: (17742, 138)\n",
      "Train: (11887, 138) Test: (2927, 138) Validation (2928, 138)\n",
      "Task2/107_1_2.csv\n",
      "Full: (14499, 138)\n",
      "Train: (9714, 138) Test: (2392, 138) Validation (2393, 138)\n",
      "Task2/108_1_2.csv\n",
      "Full: (14330, 138)\n",
      "Train: (9601, 138) Test: (2364, 138) Validation (2365, 138)\n",
      "Task2/10_0_2.csv\n",
      "Full: (10801, 138)\n",
      "Train: (7236, 138) Test: (1782, 138) Validation (1783, 138)\n",
      "Task2/15_0_2.csv\n",
      "Full: (15587, 138)\n",
      "Train: (10443, 138) Test: (2572, 138) Validation (2572, 138)\n",
      "Task2/16_0_2.csv\n",
      "Full: (21664, 138)\n",
      "Train: (14514, 138) Test: (3575, 138) Validation (3575, 138)\n",
      "Task2/17_1_2.csv\n",
      "Full: (13173, 138)\n",
      "Train: (8825, 138) Test: (2174, 138) Validation (2174, 138)\n",
      "Task2/18_1_2.csv\n",
      "Full: (14611, 138)\n",
      "Train: (9789, 138) Test: (2411, 138) Validation (2411, 138)\n",
      "Task2/23_1_2.csv\n",
      "Full: (751, 138)\n",
      "Train: (503, 138) Test: (124, 138) Validation (124, 138)\n",
      "Task2/24_1_2.csv\n",
      "Full: (11785, 138)\n",
      "Train: (7895, 138) Test: (1945, 138) Validation (1945, 138)\n",
      "Task2/25_1_2.csv\n",
      "Full: (8143, 138)\n",
      "Train: (5455, 138) Test: (1344, 138) Validation (1344, 138)\n",
      "Task2/26_1_2.csv\n",
      "Full: (9587, 138)\n",
      "Train: (6423, 138) Test: (1582, 138) Validation (1582, 138)\n",
      "Task2/27_0_2.csv\n",
      "Full: (9219, 138)\n",
      "Train: (6176, 138) Test: (1521, 138) Validation (1522, 138)\n",
      "Task2/28_0_2.csv\n",
      "Full: (8977, 138)\n",
      "Train: (6014, 138) Test: (1481, 138) Validation (1482, 138)\n",
      "Task2/29_1_2.csv\n",
      "Full: (8425, 138)\n",
      "Train: (5644, 138) Test: (1390, 138) Validation (1391, 138)\n",
      "Task2/30_1_2.csv\n",
      "Full: (8760, 138)\n",
      "Train: (5869, 138) Test: (1445, 138) Validation (1446, 138)\n",
      "Task2/31_1_2.csv\n",
      "Full: (13532, 138)\n",
      "Train: (9066, 138) Test: (2233, 138) Validation (2233, 138)\n",
      "Task2/32_1_2.csv\n",
      "Full: (13548, 138)\n",
      "Train: (9077, 138) Test: (2235, 138) Validation (2236, 138)\n",
      "Task2/35_1_2.csv\n",
      "Full: (5381, 138)\n",
      "Train: (3605, 138) Test: (888, 138) Validation (888, 138)\n",
      "Task2/36_1_2.csv\n",
      "Full: (9257, 138)\n",
      "Train: (6202, 138) Test: (1527, 138) Validation (1528, 138)\n",
      "Task2/37_0_2.csv\n",
      "Full: (10600, 138)\n",
      "Train: (7102, 138) Test: (1749, 138) Validation (1749, 138)\n",
      "Task2/38_0_2.csv\n",
      "Full: (10403, 138)\n",
      "Train: (6970, 138) Test: (1716, 138) Validation (1717, 138)\n",
      "Task2/39_0_2.csv\n",
      "Full: (7083, 138)\n",
      "Train: (4745, 138) Test: (1169, 138) Validation (1169, 138)\n",
      "Task2/40_0_2.csv\n",
      "Full: (7120, 138)\n",
      "Train: (4770, 138) Test: (1175, 138) Validation (1175, 138)\n",
      "Task2/41_0_2.csv\n",
      "Full: (10619, 138)\n",
      "Train: (7114, 138) Test: (1752, 138) Validation (1753, 138)\n",
      "Task2/42_0_2.csv\n",
      "Full: (10772, 138)\n",
      "Train: (7217, 138) Test: (1777, 138) Validation (1778, 138)\n",
      "Task2/43_1_2.csv\n",
      "Full: (7125, 138)\n",
      "Train: (4773, 138) Test: (1176, 138) Validation (1176, 138)\n",
      "Task2/44_1_2.csv\n",
      "Full: (6936, 138)\n",
      "Train: (4647, 138) Test: (1144, 138) Validation (1145, 138)\n",
      "Task2/45_1_2.csv\n",
      "Full: (21218, 138)\n",
      "Train: (14216, 138) Test: (3501, 138) Validation (3501, 138)\n",
      "Task2/46_1_2.csv\n",
      "Full: (22233, 138)\n",
      "Train: (14896, 138) Test: (3668, 138) Validation (3669, 138)\n",
      "Task2/47_1_2.csv\n",
      "Full: (5391, 138)\n",
      "Train: (3611, 138) Test: (890, 138) Validation (890, 138)\n",
      "Task2/48_1_2.csv\n",
      "Full: (8259, 138)\n",
      "Train: (5533, 138) Test: (1363, 138) Validation (1363, 138)\n",
      "Task2/49_1_2.csv\n",
      "Full: (7263, 138)\n",
      "Train: (4866, 138) Test: (1198, 138) Validation (1199, 138)\n",
      "Task2/50_1_2.csv\n",
      "Full: (9243, 138)\n",
      "Train: (6192, 138) Test: (1525, 138) Validation (1526, 138)\n",
      "Task2/51_1_2.csv\n",
      "Full: (10022, 138)\n",
      "Train: (6714, 138) Test: (1654, 138) Validation (1654, 138)\n",
      "Task2/52_1_2.csv\n",
      "Full: (10258, 138)\n",
      "Train: (6872, 138) Test: (1693, 138) Validation (1693, 138)\n",
      "Task2/55_0_2.csv\n",
      "Full: (5344, 138)\n",
      "Train: (3580, 138) Test: (882, 138) Validation (882, 138)\n",
      "Task2/56_0_2.csv\n",
      "Full: (15356, 138)\n",
      "Train: (10288, 138) Test: (2534, 138) Validation (2534, 138)\n",
      "Task2/59_1_2.csv\n",
      "Full: (11074, 138)\n",
      "Train: (7419, 138) Test: (1827, 138) Validation (1828, 138)\n",
      "Task2/60_1_2.csv\n",
      "Full: (11073, 138)\n",
      "Train: (7418, 138) Test: (1827, 138) Validation (1828, 138)\n",
      "Task2/63_0_2.csv\n",
      "Full: (12303, 138)\n",
      "Train: (8243, 138) Test: (2030, 138) Validation (2030, 138)\n",
      "Task2/64_0_2.csv\n",
      "Full: (12263, 138)\n",
      "Train: (8216, 138) Test: (2023, 138) Validation (2024, 138)\n",
      "Task2/65_0_2.csv\n",
      "Full: (16717, 138)\n",
      "Train: (11200, 138) Test: (2758, 138) Validation (2759, 138)\n",
      "Task2/66_0_2.csv\n",
      "Full: (16240, 138)\n",
      "Train: (10880, 138) Test: (2680, 138) Validation (2680, 138)\n",
      "Task2/70_0_2.csv\n",
      "Full: (7535, 138)\n",
      "Train: (5048, 138) Test: (1243, 138) Validation (1244, 138)\n",
      "Task2/72_0_2.csv\n",
      "Full: (9409, 138)\n",
      "Train: (6304, 138) Test: (1552, 138) Validation (1553, 138)\n",
      "Task2/73_1_2.csv\n",
      "Full: (6969, 138)\n",
      "Train: (4669, 138) Test: (1150, 138) Validation (1150, 138)\n",
      "Task2/77_0_2.csv\n",
      "Full: (6255, 138)\n",
      "Train: (4190, 138) Test: (1032, 138) Validation (1033, 138)\n",
      "Task2/87_1_2.csv\n",
      "Full: (10196, 138)\n",
      "Train: (6831, 138) Test: (1682, 138) Validation (1683, 138)\n",
      "Task2/88_1_2.csv\n",
      "Full: (10197, 138)\n",
      "Train: (6831, 138) Test: (1683, 138) Validation (1683, 138)\n",
      "Task2/95_1_2.csv\n",
      "Full: (8919, 138)\n",
      "Train: (5975, 138) Test: (1472, 138) Validation (1472, 138)\n",
      "Task2/96_1_2.csv\n",
      "Full: (8907, 138)\n",
      "Train: (5967, 138) Test: (1470, 138) Validation (1470, 138)\n",
      "Task2/97_0_2.csv\n",
      "Full: (7403, 138)\n",
      "Train: (4960, 138) Test: (1221, 138) Validation (1222, 138)\n",
      "Task2/98_0_2.csv\n",
      "Full: (7389, 138)\n",
      "Train: (4950, 138) Test: (1219, 138) Validation (1220, 138)\n",
      "Task3/01_0_3.csv\n",
      "Full: (9449, 138)\n",
      "Train: (6330, 138) Test: (1559, 138) Validation (1560, 138)\n",
      "Task3/02_0_3.csv\n",
      "Full: (9309, 138)\n",
      "Train: (6237, 138) Test: (1536, 138) Validation (1536, 138)\n",
      "Task3/03_1_3.csv\n",
      "Full: (8946, 138)\n",
      "Train: (5993, 138) Test: (1476, 138) Validation (1477, 138)\n",
      "Task3/04_1_3.csv\n",
      "Full: (8885, 138)\n",
      "Train: (5952, 138) Test: (1466, 138) Validation (1467, 138)\n",
      "Task3/07_0_3.csv\n",
      "Full: (8488, 138)\n",
      "Train: (5686, 138) Test: (1401, 138) Validation (1401, 138)\n",
      "Task3/08_0_3.csv\n",
      "Full: (9032, 138)\n",
      "Train: (6051, 138) Test: (1490, 138) Validation (1491, 138)\n",
      "Task3/09_0_3.csv\n",
      "Full: (8958, 138)\n",
      "Train: (6001, 138) Test: (1478, 138) Validation (1479, 138)\n",
      "Task3/100_0_3.csv\n",
      "Full: (9195, 138)\n",
      "Train: (6160, 138) Test: (1517, 138) Validation (1518, 138)\n",
      "Task3/101_0_3.csv\n",
      "Full: (9428, 138)\n",
      "Train: (6316, 138) Test: (1556, 138) Validation (1556, 138)\n",
      "Task3/102_0_3.csv\n",
      "Full: (9446, 138)\n",
      "Train: (6328, 138) Test: (1559, 138) Validation (1559, 138)\n",
      "Task3/103_1_3.csv\n",
      "Full: (8401, 138)\n",
      "Train: (5628, 138) Test: (1386, 138) Validation (1387, 138)\n",
      "Task3/104_1_3.csv\n",
      "Full: (8404, 138)\n",
      "Train: (5630, 138) Test: (1387, 138) Validation (1387, 138)\n",
      "Task3/105_0_3.csv\n",
      "Full: (9325, 138)\n",
      "Train: (6247, 138) Test: (1539, 138) Validation (1539, 138)\n",
      "Task3/106_0_3.csv\n",
      "Full: (9345, 138)\n",
      "Train: (6261, 138) Test: (1542, 138) Validation (1542, 138)\n",
      "Task3/10_0_3.csv\n",
      "Full: (8843, 138)\n",
      "Train: (5924, 138) Test: (1459, 138) Validation (1460, 138)\n",
      "Task3/11_0_3.csv\n",
      "Full: (9492, 138)\n",
      "Train: (6359, 138) Test: (1566, 138) Validation (1567, 138)\n",
      "Task3/12_0_3.csv\n",
      "Full: (9175, 138)\n",
      "Train: (6147, 138) Test: (1514, 138) Validation (1514, 138)\n",
      "Task3/13_1_3.csv\n",
      "Full: (8995, 138)\n",
      "Train: (6026, 138) Test: (1484, 138) Validation (1485, 138)\n",
      "Task3/14_1_3.csv\n",
      "Full: (8998, 138)\n",
      "Train: (6028, 138) Test: (1485, 138) Validation (1485, 138)\n",
      "Task3/15_0_3.csv\n",
      "Full: (5425, 138)\n",
      "Train: (3634, 138) Test: (895, 138) Validation (896, 138)\n",
      "Task3/16_0_3.csv\n",
      "Full: (1813, 138)\n",
      "Train: (1214, 138) Test: (299, 138) Validation (300, 138)\n",
      "Task3/17_1_3.csv\n",
      "Full: (8307, 138)\n",
      "Train: (5565, 138) Test: (1371, 138) Validation (1371, 138)\n",
      "Task3/18_1_3.csv\n",
      "Full: (9037, 138)\n",
      "Train: (6054, 138) Test: (1491, 138) Validation (1492, 138)\n",
      "Task3/23_1_3.csv\n",
      "Full: (8253, 138)\n",
      "Train: (5529, 138) Test: (1362, 138) Validation (1362, 138)\n",
      "Task3/24_1_3.csv\n",
      "Full: (8817, 138)\n",
      "Train: (5907, 138) Test: (1455, 138) Validation (1455, 138)\n",
      "Task3/25_1_3.csv\n",
      "Full: (8921, 138)\n",
      "Train: (5977, 138) Test: (1472, 138) Validation (1472, 138)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Task3/26_1_3.csv\n",
      "Full: (8963, 138)\n",
      "Train: (6005, 138) Test: (1479, 138) Validation (1479, 138)\n",
      "Task3/27_0_3.csv\n",
      "Full: (9215, 138)\n",
      "Train: (6174, 138) Test: (1520, 138) Validation (1521, 138)\n",
      "Task3/28_0_3.csv\n",
      "Full: (9178, 138)\n",
      "Train: (6149, 138) Test: (1514, 138) Validation (1515, 138)\n",
      "Task3/29_1_3.csv\n",
      "Full: (8835, 138)\n",
      "Train: (5919, 138) Test: (1458, 138) Validation (1458, 138)\n",
      "Task3/30_1_3.csv\n",
      "Full: (8856, 138)\n",
      "Train: (5933, 138) Test: (1461, 138) Validation (1462, 138)\n",
      "Task3/31_1_3.csv\n",
      "Full: (8930, 138)\n",
      "Train: (5983, 138) Test: (1473, 138) Validation (1474, 138)\n",
      "Task3/32_1_3.csv\n",
      "Full: (8824, 138)\n",
      "Train: (5912, 138) Test: (1456, 138) Validation (1456, 138)\n",
      "Task3/35_1_3.csv\n",
      "Full: (9009, 138)\n",
      "Train: (6036, 138) Test: (1486, 138) Validation (1487, 138)\n",
      "Task3/36_1_3.csv\n",
      "Full: (5637, 138)\n",
      "Train: (3776, 138) Test: (930, 138) Validation (931, 138)\n",
      "Task3/37_0_3.csv\n",
      "Full: (8967, 138)\n",
      "Train: (6007, 138) Test: (1480, 138) Validation (1480, 138)\n",
      "Task3/38_0_3.csv\n",
      "Full: (8930, 138)\n",
      "Train: (5983, 138) Test: (1473, 138) Validation (1474, 138)\n",
      "Task3/39_0_3.csv\n",
      "Full: (8708, 138)\n",
      "Train: (5834, 138) Test: (1437, 138) Validation (1437, 138)\n",
      "Task3/40_0_3.csv\n",
      "Full: (8819, 138)\n",
      "Train: (5908, 138) Test: (1455, 138) Validation (1456, 138)\n",
      "Task3/41_0_3.csv\n",
      "Full: (8867, 138)\n",
      "Train: (5940, 138) Test: (1463, 138) Validation (1464, 138)\n",
      "Task3/42_0_3.csv\n",
      "Full: (9051, 138)\n",
      "Train: (6064, 138) Test: (1493, 138) Validation (1494, 138)\n",
      "Task3/43_1_3.csv\n",
      "Full: (9722, 138)\n",
      "Train: (6513, 138) Test: (1604, 138) Validation (1605, 138)\n",
      "Task3/44_1_3.csv\n",
      "Full: (9627, 138)\n",
      "Train: (6450, 138) Test: (1588, 138) Validation (1589, 138)\n",
      "Task3/45_1_3.csv\n",
      "Full: (8919, 138)\n",
      "Train: (5975, 138) Test: (1472, 138) Validation (1472, 138)\n",
      "Task3/46_1_3.csv\n",
      "Full: (8951, 138)\n",
      "Train: (5997, 138) Test: (1477, 138) Validation (1477, 138)\n",
      "Task3/47_1_3.csv\n",
      "Full: (7491, 138)\n",
      "Train: (5018, 138) Test: (1236, 138) Validation (1237, 138)\n",
      "Task3/48_1_3.csv\n",
      "Full: (7338, 138)\n",
      "Train: (4916, 138) Test: (1211, 138) Validation (1211, 138)\n",
      "Task3/49_1_3.csv\n",
      "Full: (7807, 138)\n",
      "Train: (5230, 138) Test: (1288, 138) Validation (1289, 138)\n",
      "Task3/50_1_3.csv\n",
      "Full: (7314, 138)\n",
      "Train: (4900, 138) Test: (1207, 138) Validation (1207, 138)\n",
      "Task3/51_1_3.csv\n",
      "Full: (7515, 138)\n",
      "Train: (5035, 138) Test: (1240, 138) Validation (1240, 138)\n",
      "Task3/52_1_3.csv\n",
      "Full: (8977, 138)\n",
      "Train: (6014, 138) Test: (1481, 138) Validation (1482, 138)\n",
      "Task3/55_0_3.csv\n",
      "Full: (9581, 138)\n",
      "Train: (6419, 138) Test: (1581, 138) Validation (1581, 138)\n",
      "Task3/56_0_3.csv\n",
      "Full: (7663, 138)\n",
      "Train: (5134, 138) Test: (1264, 138) Validation (1265, 138)\n",
      "Task3/57_0_3P2.csv\n",
      "Full: (2650, 138)\n",
      "Train: (1775, 138) Test: (437, 138) Validation (438, 138)\n",
      "Task3/58_0_3.csv\n",
      "Full: (5056, 138)\n",
      "Train: (3387, 138) Test: (834, 138) Validation (835, 138)\n",
      "Task3/59_1_3.csv\n",
      "Full: (6591, 138)\n",
      "Train: (4415, 138) Test: (1088, 138) Validation (1088, 138)\n",
      "Task3/60_1_3.csv\n",
      "Full: (9003, 138)\n",
      "Train: (6032, 138) Test: (1485, 138) Validation (1486, 138)\n",
      "Task3/63_0_3.csv\n",
      "Full: (9514, 138)\n",
      "Train: (6374, 138) Test: (1570, 138) Validation (1570, 138)\n",
      "Task3/64_0_3.csv\n",
      "Full: (9507, 138)\n",
      "Train: (6369, 138) Test: (1569, 138) Validation (1569, 138)\n",
      "Task3/65_0_3.csv\n",
      "Full: (9353, 138)\n",
      "Train: (6266, 138) Test: (1543, 138) Validation (1544, 138)\n",
      "Task3/66_0_3.csv\n",
      "Full: (9361, 138)\n",
      "Train: (6271, 138) Test: (1545, 138) Validation (1545, 138)\n",
      "Task3/73_1_3.csv\n",
      "Full: (6697, 138)\n",
      "Train: (4486, 138) Test: (1105, 138) Validation (1106, 138)\n",
      "Task3/74_1_3.csv\n",
      "Full: (9724, 138)\n",
      "Train: (6515, 138) Test: (1604, 138) Validation (1605, 138)\n",
      "Task3/75_1_3.csv\n",
      "Full: (7045, 138)\n",
      "Train: (4720, 138) Test: (1162, 138) Validation (1163, 138)\n",
      "Task3/76_1_3.csv\n",
      "Full: (7724, 138)\n",
      "Train: (5175, 138) Test: (1274, 138) Validation (1275, 138)\n",
      "Task3/77_0_3.csv\n",
      "Full: (7536, 138)\n",
      "Train: (5049, 138) Test: (1243, 138) Validation (1244, 138)\n",
      "Task3/78_0_3.csv\n",
      "Full: (7536, 138)\n",
      "Train: (5049, 138) Test: (1243, 138) Validation (1244, 138)\n",
      "Task3/79_1_3.csv\n",
      "Full: (10030, 138)\n",
      "Train: (6720, 138) Test: (1655, 138) Validation (1655, 138)\n",
      "Task3/80_1_3.csv\n",
      "Full: (9976, 138)\n",
      "Train: (6683, 138) Test: (1646, 138) Validation (1647, 138)\n",
      "Task3/81_1_3.csv\n",
      "Full: (4070, 138)\n",
      "Train: (2726, 138) Test: (672, 138) Validation (672, 138)\n",
      "Task3/82_1_3.csv\n",
      "Full: (7680, 138)\n",
      "Train: (5145, 138) Test: (1267, 138) Validation (1268, 138)\n",
      "Task3/83_0_3.csv\n",
      "Full: (7460, 138)\n",
      "Train: (4998, 138) Test: (1231, 138) Validation (1231, 138)\n",
      "Task3/84_0_3.csv\n",
      "Full: (7474, 138)\n",
      "Train: (5007, 138) Test: (1233, 138) Validation (1234, 138)\n",
      "Task3/87_1_3.csv\n",
      "Full: (9336, 138)\n",
      "Train: (6255, 138) Test: (1540, 138) Validation (1541, 138)\n",
      "Task3/88_1_3.csv\n",
      "Full: (9333, 138)\n",
      "Train: (6253, 138) Test: (1540, 138) Validation (1540, 138)\n",
      "Task3/95_1_3.csv\n",
      "Full: (5400, 138)\n",
      "Train: (3618, 138) Test: (891, 138) Validation (891, 138)\n",
      "\n",
      "\n",
      "FINAL\n",
      "Train: (1227083, 138) Test: (302221, 138) Validation (302327, 138)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "indicies = ['frame'] + ['x_{0}'.format(i) for i in range(0,68)] + ['y_{0}'.format(i) for i in range(0,68)]\n",
    "\n",
    "flag = True\n",
    "Train = None\n",
    "Test = None\n",
    "Valid = None\n",
    "\n",
    "for i in tqdm(range(0,len(files))):\n",
    "    \n",
    "    f = files[i]\n",
    "    \n",
    "    # Load CSV\n",
    "    print(f)\n",
    "    df = pd.read_csv(\n",
    "            os.path.join(path,f), \n",
    "            error_bad_lines=False, \n",
    "            warn_bad_lines=True)\n",
    "    df.columns = df.columns.str.replace(' ', '')\n",
    "    \n",
    "    try:\n",
    "        face_lmk = df[indicies]\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        print(list(df.columns.values))\n",
    "    \n",
    "    # Filter NaN rows\n",
    "    nans = df[df[indicies].isnull().any(axis=1)]\n",
    "    if not nans.empty:\n",
    "        dropList = nans.index.tolist()\n",
    "        for r in dropList:\n",
    "            warnings.warn('Dropping row due to NaN, {0}'.format(r))\n",
    "        face_lmk.drop(dropList, inplace=True)\n",
    "        \n",
    "    face_lmk.insert(0,'filename',[f for i in range(0,face_lmk.shape[0])],True)\n",
    "        \n",
    "    print('Full:',face_lmk.shape)\n",
    "    \n",
    "    # Create Training, Test, Validation subsets\n",
    "    train, tmp = train_test_split(face_lmk, test_size=0.33)\n",
    "    test, val = train_test_split(tmp, test_size=0.5)\n",
    "    print('Train:',train.shape,'Test:',test.shape,'Validation',val.shape)\n",
    "    \n",
    "    if flag:\n",
    "        flag = False\n",
    "        Train = train\n",
    "        Test = test\n",
    "        Valid = val\n",
    "    else:\n",
    "        Train = Train.append(train)\n",
    "        Test = Test.append(test)\n",
    "        Valid = Valid.append(val)\n",
    "\n",
    "# Final Dataset (shuffled)\n",
    "Train = Train.sample(frac=1).reset_index(drop=True)\n",
    "Test = Test.sample(frac=1).reset_index(drop=True)\n",
    "Valid = Valid.sample(frac=1).reset_index(drop=True)\n",
    "\n",
    "print('\\n\\nFINAL')\n",
    "print('Train:',Train.shape,'Test:',Test.shape,'Validation',Valid.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>frame</th>\n",
       "      <th>x_0</th>\n",
       "      <th>x_1</th>\n",
       "      <th>x_2</th>\n",
       "      <th>x_3</th>\n",
       "      <th>x_4</th>\n",
       "      <th>x_5</th>\n",
       "      <th>x_6</th>\n",
       "      <th>x_7</th>\n",
       "      <th>...</th>\n",
       "      <th>y_58</th>\n",
       "      <th>y_59</th>\n",
       "      <th>y_60</th>\n",
       "      <th>y_61</th>\n",
       "      <th>y_62</th>\n",
       "      <th>y_63</th>\n",
       "      <th>y_64</th>\n",
       "      <th>y_65</th>\n",
       "      <th>y_66</th>\n",
       "      <th>y_67</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Task3/60_1_3.csv</td>\n",
       "      <td>1373</td>\n",
       "      <td>404.555</td>\n",
       "      <td>401.430</td>\n",
       "      <td>400.485</td>\n",
       "      <td>402.306</td>\n",
       "      <td>407.210</td>\n",
       "      <td>416.870</td>\n",
       "      <td>429.420</td>\n",
       "      <td>444.360</td>\n",
       "      <td>...</td>\n",
       "      <td>628.345</td>\n",
       "      <td>620.622</td>\n",
       "      <td>607.475</td>\n",
       "      <td>611.181</td>\n",
       "      <td>613.607</td>\n",
       "      <td>614.268</td>\n",
       "      <td>615.297</td>\n",
       "      <td>620.542</td>\n",
       "      <td>620.016</td>\n",
       "      <td>617.400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Task3/76_1_3.csv</td>\n",
       "      <td>7602</td>\n",
       "      <td>237.776</td>\n",
       "      <td>237.752</td>\n",
       "      <td>240.315</td>\n",
       "      <td>244.228</td>\n",
       "      <td>252.111</td>\n",
       "      <td>267.379</td>\n",
       "      <td>287.687</td>\n",
       "      <td>311.492</td>\n",
       "      <td>...</td>\n",
       "      <td>685.459</td>\n",
       "      <td>676.209</td>\n",
       "      <td>660.563</td>\n",
       "      <td>657.141</td>\n",
       "      <td>659.990</td>\n",
       "      <td>660.159</td>\n",
       "      <td>666.139</td>\n",
       "      <td>670.698</td>\n",
       "      <td>670.479</td>\n",
       "      <td>667.159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Task2/44_1_2.csv</td>\n",
       "      <td>4029</td>\n",
       "      <td>775.189</td>\n",
       "      <td>771.471</td>\n",
       "      <td>772.501</td>\n",
       "      <td>776.772</td>\n",
       "      <td>777.276</td>\n",
       "      <td>778.777</td>\n",
       "      <td>780.727</td>\n",
       "      <td>786.340</td>\n",
       "      <td>...</td>\n",
       "      <td>473.380</td>\n",
       "      <td>468.392</td>\n",
       "      <td>460.129</td>\n",
       "      <td>456.621</td>\n",
       "      <td>458.218</td>\n",
       "      <td>458.067</td>\n",
       "      <td>463.795</td>\n",
       "      <td>460.822</td>\n",
       "      <td>460.339</td>\n",
       "      <td>457.962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Task1/36_1_1.csv</td>\n",
       "      <td>3715</td>\n",
       "      <td>784.563</td>\n",
       "      <td>783.634</td>\n",
       "      <td>787.208</td>\n",
       "      <td>794.842</td>\n",
       "      <td>802.974</td>\n",
       "      <td>813.964</td>\n",
       "      <td>824.780</td>\n",
       "      <td>841.008</td>\n",
       "      <td>...</td>\n",
       "      <td>515.601</td>\n",
       "      <td>509.953</td>\n",
       "      <td>499.575</td>\n",
       "      <td>501.388</td>\n",
       "      <td>502.587</td>\n",
       "      <td>502.292</td>\n",
       "      <td>503.342</td>\n",
       "      <td>503.797</td>\n",
       "      <td>503.625</td>\n",
       "      <td>502.357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Task3/39_0_3.csv</td>\n",
       "      <td>589</td>\n",
       "      <td>672.413</td>\n",
       "      <td>686.212</td>\n",
       "      <td>703.018</td>\n",
       "      <td>724.545</td>\n",
       "      <td>759.962</td>\n",
       "      <td>798.916</td>\n",
       "      <td>837.138</td>\n",
       "      <td>883.660</td>\n",
       "      <td>...</td>\n",
       "      <td>816.567</td>\n",
       "      <td>820.823</td>\n",
       "      <td>814.941</td>\n",
       "      <td>799.056</td>\n",
       "      <td>793.615</td>\n",
       "      <td>785.255</td>\n",
       "      <td>772.065</td>\n",
       "      <td>790.053</td>\n",
       "      <td>799.035</td>\n",
       "      <td>804.719</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 138 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           filename  frame      x_0      x_1      x_2      x_3      x_4  \\\n",
       "0  Task3/60_1_3.csv   1373  404.555  401.430  400.485  402.306  407.210   \n",
       "1  Task3/76_1_3.csv   7602  237.776  237.752  240.315  244.228  252.111   \n",
       "2  Task2/44_1_2.csv   4029  775.189  771.471  772.501  776.772  777.276   \n",
       "3  Task1/36_1_1.csv   3715  784.563  783.634  787.208  794.842  802.974   \n",
       "4  Task3/39_0_3.csv    589  672.413  686.212  703.018  724.545  759.962   \n",
       "\n",
       "       x_5      x_6      x_7  ...     y_58     y_59     y_60     y_61  \\\n",
       "0  416.870  429.420  444.360  ...  628.345  620.622  607.475  611.181   \n",
       "1  267.379  287.687  311.492  ...  685.459  676.209  660.563  657.141   \n",
       "2  778.777  780.727  786.340  ...  473.380  468.392  460.129  456.621   \n",
       "3  813.964  824.780  841.008  ...  515.601  509.953  499.575  501.388   \n",
       "4  798.916  837.138  883.660  ...  816.567  820.823  814.941  799.056   \n",
       "\n",
       "      y_62     y_63     y_64     y_65     y_66     y_67  \n",
       "0  613.607  614.268  615.297  620.542  620.016  617.400  \n",
       "1  659.990  660.159  666.139  670.698  670.479  667.159  \n",
       "2  458.218  458.067  463.795  460.822  460.339  457.962  \n",
       "3  502.587  502.292  503.342  503.797  503.625  502.357  \n",
       "4  793.615  785.255  772.065  790.053  799.035  804.719  \n",
       "\n",
       "[5 rows x 138 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>frame</th>\n",
       "      <th>x_0</th>\n",
       "      <th>x_1</th>\n",
       "      <th>x_2</th>\n",
       "      <th>x_3</th>\n",
       "      <th>x_4</th>\n",
       "      <th>x_5</th>\n",
       "      <th>x_6</th>\n",
       "      <th>x_7</th>\n",
       "      <th>...</th>\n",
       "      <th>y_58</th>\n",
       "      <th>y_59</th>\n",
       "      <th>y_60</th>\n",
       "      <th>y_61</th>\n",
       "      <th>y_62</th>\n",
       "      <th>y_63</th>\n",
       "      <th>y_64</th>\n",
       "      <th>y_65</th>\n",
       "      <th>y_66</th>\n",
       "      <th>y_67</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Task2/87_1_2.csv</td>\n",
       "      <td>3392</td>\n",
       "      <td>430.175</td>\n",
       "      <td>430.126</td>\n",
       "      <td>433.957</td>\n",
       "      <td>440.151</td>\n",
       "      <td>444.710</td>\n",
       "      <td>450.061</td>\n",
       "      <td>456.041</td>\n",
       "      <td>463.564</td>\n",
       "      <td>...</td>\n",
       "      <td>439.267</td>\n",
       "      <td>436.095</td>\n",
       "      <td>429.500</td>\n",
       "      <td>426.723</td>\n",
       "      <td>427.142</td>\n",
       "      <td>426.072</td>\n",
       "      <td>426.632</td>\n",
       "      <td>428.179</td>\n",
       "      <td>429.062</td>\n",
       "      <td>428.156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Task2/56_0_2.csv</td>\n",
       "      <td>8562</td>\n",
       "      <td>1156.890</td>\n",
       "      <td>1160.150</td>\n",
       "      <td>1168.360</td>\n",
       "      <td>1182.680</td>\n",
       "      <td>1200.430</td>\n",
       "      <td>1223.250</td>\n",
       "      <td>1250.300</td>\n",
       "      <td>1282.170</td>\n",
       "      <td>...</td>\n",
       "      <td>676.593</td>\n",
       "      <td>688.996</td>\n",
       "      <td>699.576</td>\n",
       "      <td>669.012</td>\n",
       "      <td>664.380</td>\n",
       "      <td>660.769</td>\n",
       "      <td>686.944</td>\n",
       "      <td>667.031</td>\n",
       "      <td>670.631</td>\n",
       "      <td>675.103</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Task2/24_1_2.csv</td>\n",
       "      <td>2568</td>\n",
       "      <td>620.756</td>\n",
       "      <td>620.302</td>\n",
       "      <td>624.151</td>\n",
       "      <td>633.391</td>\n",
       "      <td>646.784</td>\n",
       "      <td>663.190</td>\n",
       "      <td>682.528</td>\n",
       "      <td>704.926</td>\n",
       "      <td>...</td>\n",
       "      <td>490.404</td>\n",
       "      <td>477.492</td>\n",
       "      <td>453.228</td>\n",
       "      <td>454.369</td>\n",
       "      <td>457.153</td>\n",
       "      <td>455.203</td>\n",
       "      <td>456.220</td>\n",
       "      <td>473.905</td>\n",
       "      <td>475.533</td>\n",
       "      <td>471.680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Task1/15_0_1.csv</td>\n",
       "      <td>6170</td>\n",
       "      <td>777.888</td>\n",
       "      <td>772.380</td>\n",
       "      <td>771.702</td>\n",
       "      <td>776.387</td>\n",
       "      <td>780.935</td>\n",
       "      <td>786.789</td>\n",
       "      <td>793.432</td>\n",
       "      <td>803.042</td>\n",
       "      <td>...</td>\n",
       "      <td>580.831</td>\n",
       "      <td>578.565</td>\n",
       "      <td>572.691</td>\n",
       "      <td>568.731</td>\n",
       "      <td>568.789</td>\n",
       "      <td>567.455</td>\n",
       "      <td>568.925</td>\n",
       "      <td>569.478</td>\n",
       "      <td>570.065</td>\n",
       "      <td>569.232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Task1/52_1_1.csv</td>\n",
       "      <td>2779</td>\n",
       "      <td>811.952</td>\n",
       "      <td>812.464</td>\n",
       "      <td>816.311</td>\n",
       "      <td>825.059</td>\n",
       "      <td>834.357</td>\n",
       "      <td>850.497</td>\n",
       "      <td>866.280</td>\n",
       "      <td>886.488</td>\n",
       "      <td>...</td>\n",
       "      <td>543.764</td>\n",
       "      <td>537.051</td>\n",
       "      <td>526.041</td>\n",
       "      <td>528.072</td>\n",
       "      <td>531.184</td>\n",
       "      <td>531.406</td>\n",
       "      <td>528.463</td>\n",
       "      <td>531.695</td>\n",
       "      <td>531.729</td>\n",
       "      <td>528.364</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 138 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           filename  frame       x_0       x_1       x_2       x_3       x_4  \\\n",
       "0  Task2/87_1_2.csv   3392   430.175   430.126   433.957   440.151   444.710   \n",
       "1  Task2/56_0_2.csv   8562  1156.890  1160.150  1168.360  1182.680  1200.430   \n",
       "2  Task2/24_1_2.csv   2568   620.756   620.302   624.151   633.391   646.784   \n",
       "3  Task1/15_0_1.csv   6170   777.888   772.380   771.702   776.387   780.935   \n",
       "4  Task1/52_1_1.csv   2779   811.952   812.464   816.311   825.059   834.357   \n",
       "\n",
       "        x_5       x_6       x_7  ...     y_58     y_59     y_60     y_61  \\\n",
       "0   450.061   456.041   463.564  ...  439.267  436.095  429.500  426.723   \n",
       "1  1223.250  1250.300  1282.170  ...  676.593  688.996  699.576  669.012   \n",
       "2   663.190   682.528   704.926  ...  490.404  477.492  453.228  454.369   \n",
       "3   786.789   793.432   803.042  ...  580.831  578.565  572.691  568.731   \n",
       "4   850.497   866.280   886.488  ...  543.764  537.051  526.041  528.072   \n",
       "\n",
       "      y_62     y_63     y_64     y_65     y_66     y_67  \n",
       "0  427.142  426.072  426.632  428.179  429.062  428.156  \n",
       "1  664.380  660.769  686.944  667.031  670.631  675.103  \n",
       "2  457.153  455.203  456.220  473.905  475.533  471.680  \n",
       "3  568.789  567.455  568.925  569.478  570.065  569.232  \n",
       "4  531.184  531.406  528.463  531.695  531.729  528.364  \n",
       "\n",
       "[5 rows x 138 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>frame</th>\n",
       "      <th>x_0</th>\n",
       "      <th>x_1</th>\n",
       "      <th>x_2</th>\n",
       "      <th>x_3</th>\n",
       "      <th>x_4</th>\n",
       "      <th>x_5</th>\n",
       "      <th>x_6</th>\n",
       "      <th>x_7</th>\n",
       "      <th>...</th>\n",
       "      <th>y_58</th>\n",
       "      <th>y_59</th>\n",
       "      <th>y_60</th>\n",
       "      <th>y_61</th>\n",
       "      <th>y_62</th>\n",
       "      <th>y_63</th>\n",
       "      <th>y_64</th>\n",
       "      <th>y_65</th>\n",
       "      <th>y_66</th>\n",
       "      <th>y_67</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Task3/11_0_3.csv</td>\n",
       "      <td>3836</td>\n",
       "      <td>418.719</td>\n",
       "      <td>420.463</td>\n",
       "      <td>423.483</td>\n",
       "      <td>427.405</td>\n",
       "      <td>432.382</td>\n",
       "      <td>438.960</td>\n",
       "      <td>446.259</td>\n",
       "      <td>453.954</td>\n",
       "      <td>...</td>\n",
       "      <td>403.213</td>\n",
       "      <td>402.832</td>\n",
       "      <td>399.300</td>\n",
       "      <td>396.289</td>\n",
       "      <td>395.733</td>\n",
       "      <td>394.144</td>\n",
       "      <td>392.556</td>\n",
       "      <td>395.651</td>\n",
       "      <td>397.218</td>\n",
       "      <td>397.697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Task3/37_0_3.csv</td>\n",
       "      <td>7588</td>\n",
       "      <td>833.998</td>\n",
       "      <td>833.436</td>\n",
       "      <td>837.180</td>\n",
       "      <td>841.604</td>\n",
       "      <td>846.929</td>\n",
       "      <td>855.454</td>\n",
       "      <td>863.699</td>\n",
       "      <td>875.771</td>\n",
       "      <td>...</td>\n",
       "      <td>867.055</td>\n",
       "      <td>858.906</td>\n",
       "      <td>846.423</td>\n",
       "      <td>853.145</td>\n",
       "      <td>856.016</td>\n",
       "      <td>856.688</td>\n",
       "      <td>860.449</td>\n",
       "      <td>858.865</td>\n",
       "      <td>858.267</td>\n",
       "      <td>854.593</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Task3/51_1_3.csv</td>\n",
       "      <td>6962</td>\n",
       "      <td>779.146</td>\n",
       "      <td>778.372</td>\n",
       "      <td>782.651</td>\n",
       "      <td>791.977</td>\n",
       "      <td>802.551</td>\n",
       "      <td>815.555</td>\n",
       "      <td>832.259</td>\n",
       "      <td>854.179</td>\n",
       "      <td>...</td>\n",
       "      <td>1021.160</td>\n",
       "      <td>1019.110</td>\n",
       "      <td>1006.040</td>\n",
       "      <td>1008.240</td>\n",
       "      <td>1008.800</td>\n",
       "      <td>1006.910</td>\n",
       "      <td>1001.710</td>\n",
       "      <td>1008.470</td>\n",
       "      <td>1009.380</td>\n",
       "      <td>1007.770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Task3/60_1_3.csv</td>\n",
       "      <td>3970</td>\n",
       "      <td>352.368</td>\n",
       "      <td>353.072</td>\n",
       "      <td>355.795</td>\n",
       "      <td>360.580</td>\n",
       "      <td>368.474</td>\n",
       "      <td>380.350</td>\n",
       "      <td>394.062</td>\n",
       "      <td>408.004</td>\n",
       "      <td>...</td>\n",
       "      <td>607.093</td>\n",
       "      <td>602.012</td>\n",
       "      <td>593.396</td>\n",
       "      <td>596.238</td>\n",
       "      <td>597.580</td>\n",
       "      <td>596.653</td>\n",
       "      <td>592.685</td>\n",
       "      <td>597.908</td>\n",
       "      <td>598.625</td>\n",
       "      <td>597.047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Task2/97_0_2.csv</td>\n",
       "      <td>1411</td>\n",
       "      <td>734.979</td>\n",
       "      <td>738.396</td>\n",
       "      <td>746.021</td>\n",
       "      <td>756.921</td>\n",
       "      <td>776.610</td>\n",
       "      <td>803.812</td>\n",
       "      <td>832.798</td>\n",
       "      <td>865.327</td>\n",
       "      <td>...</td>\n",
       "      <td>815.280</td>\n",
       "      <td>813.997</td>\n",
       "      <td>805.553</td>\n",
       "      <td>800.371</td>\n",
       "      <td>800.034</td>\n",
       "      <td>797.212</td>\n",
       "      <td>798.662</td>\n",
       "      <td>796.653</td>\n",
       "      <td>800.105</td>\n",
       "      <td>800.838</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 138 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           filename  frame      x_0      x_1      x_2      x_3      x_4  \\\n",
       "0  Task3/11_0_3.csv   3836  418.719  420.463  423.483  427.405  432.382   \n",
       "1  Task3/37_0_3.csv   7588  833.998  833.436  837.180  841.604  846.929   \n",
       "2  Task3/51_1_3.csv   6962  779.146  778.372  782.651  791.977  802.551   \n",
       "3  Task3/60_1_3.csv   3970  352.368  353.072  355.795  360.580  368.474   \n",
       "4  Task2/97_0_2.csv   1411  734.979  738.396  746.021  756.921  776.610   \n",
       "\n",
       "       x_5      x_6      x_7  ...      y_58      y_59      y_60      y_61  \\\n",
       "0  438.960  446.259  453.954  ...   403.213   402.832   399.300   396.289   \n",
       "1  855.454  863.699  875.771  ...   867.055   858.906   846.423   853.145   \n",
       "2  815.555  832.259  854.179  ...  1021.160  1019.110  1006.040  1008.240   \n",
       "3  380.350  394.062  408.004  ...   607.093   602.012   593.396   596.238   \n",
       "4  803.812  832.798  865.327  ...   815.280   813.997   805.553   800.371   \n",
       "\n",
       "       y_62      y_63      y_64      y_65      y_66      y_67  \n",
       "0   395.733   394.144   392.556   395.651   397.218   397.697  \n",
       "1   856.016   856.688   860.449   858.865   858.267   854.593  \n",
       "2  1008.800  1006.910  1001.710  1008.470  1009.380  1007.770  \n",
       "3   597.580   596.653   592.685   597.908   598.625   597.047  \n",
       "4   800.034   797.212   798.662   796.653   800.105   800.838  \n",
       "\n",
       "[5 rows x 138 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Valid.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_path = './data/tmp_analysis/{0}'.format(task_name)\n",
    "save_train_path = save_path + '_train.csv'\n",
    "save_test_path = save_path + '_test.csv'\n",
    "save_valid_path = save_path + '_valid.csv'\n",
    "\n",
    "\n",
    "Train.to_csv(save_train_path)\n",
    "Test.to_csv(save_test_path)\n",
    "Valid.to_csv(save_valid_path)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}