scripts/02_prepare_fl_voter.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from glob import glob\n",
    "import libarchive # `pip install libarchive-c` and using with libarchive 3.4.x\n",
    "from io import BytesIO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20170207_VoterDetail.7z\n",
      "20170207_VoterDetail/ALA_20170207.txt 29.292259216308594\n",
      "20170207_VoterDetail/BAK_20170207.txt 31.640080451965332\n",
      "20170207_VoterDetail/BAY_20170207.txt 51.27226448059082\n",
      "20170207_VoterDetail/BRA_20170207.txt 53.74375057220459\n",
      "20170207_VoterDetail/BRE_20170207.txt 119.11902141571045\n",
      "20170207_VoterDetail/BRO_20170207.txt 310.09432792663574\n",
      "20170207_VoterDetail/CAL_20170207.txt 311.40102672576904\n",
      "20170207_VoterDetail/CHA_20170207.txt 331.9234809875488\n",
      "20170207_VoterDetail/CIT_20170207.txt 348.43804264068604\n",
      "20170207_VoterDetail/CLA_20170207.txt 371.96534156799316\n",
      "20170207_VoterDetail/CLL_20170207.txt 406.0159959793091\n",
      "20170207_VoterDetail/CLM_20170207.txt 412.04163455963135\n",
      "20170207_VoterDetail/DAD_20170207.txt 631.8463592529297\n",
      "20170207_VoterDetail/DES_20170207.txt 634.3178129196167\n",
      "20170207_VoterDetail/DIX_20170207.txt 635.8692283630371\n",
      "20170207_VoterDetail/DUV_20170207.txt 732.8321886062622\n",
      "20170207_VoterDetail/ESC_20170207.txt 766.6300640106201\n",
      "20170207_VoterDetail/FLA_20170207.txt 779.0231113433838\n",
      "20170207_VoterDetail/FRA_20170207.txt 780.2824745178223\n",
      "20170207_VoterDetail/GAD_20170207.txt 784.6675453186035\n",
      "20170207_VoterDetail/GIL_20170207.txt 786.3906574249268\n",
      "20170207_VoterDetail/GLA_20170207.txt 787.6066637039185\n",
      "20170207_VoterDetail/GUL_20170207.txt 789.2000017166138\n",
      "20170207_VoterDetail/HAM_20170207.txt 790.476863861084\n",
      "20170207_VoterDetail/HAR_20170207.txt 792.5194969177246\n",
      "20170207_VoterDetail/HEN_20170207.txt 795.3739643096924\n",
      "20170207_VoterDetail/HER_20170207.txt 816.2905158996582\n",
      "20170207_VoterDetail/HIG_20170207.txt 826.1917791366577\n",
      "20170207_VoterDetail/HIL_20170207.txt 959.8866882324219\n",
      "20170207_VoterDetail/HOL_20170207.txt 961.5880575180054\n",
      "20170207_VoterDetail/IND_20170207.txt 978.4171915054321\n",
      "20170207_VoterDetail/JAC_20170207.txt 982.7562532424927\n",
      "20170207_VoterDetail/JEF_20170207.txt 984.1736707687378\n",
      "20170207_VoterDetail/LAF_20170207.txt 984.8291969299316\n",
      "20170207_VoterDetail/LAK_20170207.txt 1017.4748048782349\n",
      "20170207_VoterDetail/LEE_20170207.txt 1091.3626956939697\n",
      "20170207_VoterDetail/LEO_20170207.txt 1124.4955854415894\n",
      "20170207_VoterDetail/LEV_20170207.txt 1128.6503267288208\n",
      "20170207_VoterDetail/LIB_20170207.txt 1129.35680103302\n",
      "20170207_VoterDetail/MAD_20170207.txt 1131.2379331588745\n",
      "20170207_VoterDetail/MAN_20170207.txt 1168.659688949585\n",
      "20170207_VoterDetail/MON_20170207.txt 1178.138653755188\n",
      "20170207_VoterDetail/MRN_20170207.txt 1213.4200582504272\n",
      "20170207_VoterDetail/MRT_20170207.txt 1230.4996767044067\n",
      "20170207_VoterDetail/NAS_20170207.txt 1240.2789793014526\n",
      "20170207_VoterDetail/OKA_20170207.txt 1262.3098163604736\n",
      "20170207_VoterDetail/OKE_20170207.txt 1265.3208875656128\n",
      "20170207_VoterDetail/ORA_20170207.txt 1385.5390720367432\n",
      "20170207_VoterDetail/OSC_20170207.txt 1418.8044681549072\n",
      "20170207_VoterDetail/PAL_20170207.txt 1568.3288040161133\n",
      "20170207_VoterDetail/PAS_20170207.txt 1623.5615692138672\n",
      "20170207_VoterDetail/PIN_20170207.txt 1727.3045721054077\n",
      "20170207_VoterDetail/POL_20170207.txt 1790.2020425796509\n",
      "20170207_VoterDetail/PUT_20170207.txt 1797.4803819656372\n",
      "20170207_VoterDetail/SAN_20170207.txt 1817.6592750549316\n",
      "20170207_VoterDetail/SAR_20170207.txt 1863.0826930999756\n",
      "20170207_VoterDetail/SEM_20170207.txt 1906.6497440338135\n",
      "20170207_VoterDetail/STJ_20170207.txt 1934.9651346206665\n",
      "20170207_VoterDetail/STL_20170207.txt 1967.3346490859985\n",
      "20170207_VoterDetail/SUM_20170207.txt 1981.9009437561035\n",
      "20170207_VoterDetail/SUW_20170207.txt 1985.817211151123\n",
      "20170207_VoterDetail/TAY_20170207.txt 1987.6660499572754\n",
      "20170207_VoterDetail/UNI_20170207.txt 1988.768856048584\n",
      "20170207_VoterDetail/VOL_20170207.txt 2047.3908910751343\n",
      "20170207_VoterDetail/WAK_20170207.txt 2050.4972190856934\n",
      "20170207_VoterDetail/WAL_20170207.txt 2058.0556240081787\n",
      "20170207_VoterDetail/WAS_20170207.txt 2060.3557147979736\n"
     ]
    }
   ],
   "source": [
    "total_size = 0\n",
    "header = True\n",
    "for fn in sorted(glob('*.7z')):\n",
    "    print(fn)\n",
    "    with libarchive.file_reader(fn) as e:\n",
    "        for entry in e:\n",
    "            if not entry.isfile:\n",
    "                continue\n",
    "            total_size += entry.size\n",
    "            print(entry.name, total_size / (1024*1024))\n",
    "            with BytesIO() as f:\n",
    "                for i, block in enumerate(entry.get_blocks()):\n",
    "                    f.write(block)\n",
    "                f.seek(0)\n",
    "                #df = pd.read_csv(f, nrows=10, delimiter='\\t', header=None, usecols=[2, 4, 19, 20])\n",
    "                df = pd.read_csv(f, delimiter='\\t', header=None, usecols=[2, 4, 19, 20])\n",
    "            df.columns = ['last_name', 'first_name', 'gender', 'race']\n",
    "            df.to_csv('/opt/data/fl_voter/fl_voter_name_gender_race.csv', mode='a', header=header, index=False)\n",
    "            header = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>last_name</th>\n",
       "      <th>first_name</th>\n",
       "      <th>gender</th>\n",
       "      <th>race</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Walker</td>\n",
       "      <td>Elizabeth</td>\n",
       "      <td>F</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Palmer</td>\n",
       "      <td>Alton</td>\n",
       "      <td>M</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Mc Cleod</td>\n",
       "      <td>Alicia</td>\n",
       "      <td>F</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Scarborough</td>\n",
       "      <td>Dale</td>\n",
       "      <td>M</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Walker</td>\n",
       "      <td>Daniel</td>\n",
       "      <td>M</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13710353</th>\n",
       "      <td>Walters</td>\n",
       "      <td>William</td>\n",
       "      <td>M</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13710354</th>\n",
       "      <td>Sawyer</td>\n",
       "      <td>Matthew</td>\n",
       "      <td>M</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13710355</th>\n",
       "      <td>Thomas</td>\n",
       "      <td>Janine</td>\n",
       "      <td>F</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13710356</th>\n",
       "      <td>Campbell</td>\n",
       "      <td>Angel</td>\n",
       "      <td>F</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13710357</th>\n",
       "      <td>Bruner</td>\n",
       "      <td>Jeb</td>\n",
       "      <td>M</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>13710358 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            last_name first_name gender  race\n",
       "0              Walker  Elizabeth      F     5\n",
       "1              Palmer      Alton      M     5\n",
       "2            Mc Cleod     Alicia      F     3\n",
       "3         Scarborough       Dale      M     5\n",
       "4              Walker     Daniel      M     5\n",
       "...               ...        ...    ...   ...\n",
       "13710353      Walters    William      M     5\n",
       "13710354       Sawyer    Matthew      M     5\n",
       "13710355       Thomas     Janine      F     5\n",
       "13710356     Campbell      Angel      F     7\n",
       "13710357       Bruner        Jeb      M     5\n",
       "\n",
       "[13710358 rows x 4 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('/opt/data/fl_voter/fl_voter_name_gender_race.csv')\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}