fetch_wnba.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from bs4 import BeautifulSoup\n",
    "import re\n",
    "import subprocess\n",
    "import pandas as pd\n",
    "schools = ['LVA', 'LAS', 'SEA', 'MIN', 'PHO', 'DAL', 'CON', 'WAS', 'CHI', 'NYL', 'IND', 'ATL']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "folder = 'WNBA'\n",
    "year = 2019\n",
    "\n",
    "try:\n",
    "    os.mkdir(folder)\n",
    "    print(\"Directory {} created\".format(folder)) \n",
    "except FileExistsError:\n",
    "        pass\n",
    "tables = {}\n",
    "for team in schools:\n",
    "    target = os.path.join(folder,team + str(year) + '.html')\n",
    "    print(target)\n",
    "    # get the files\n",
    "    if not os.path.exists(target):\n",
    "        subprocess.call(['wget','-O',target,\n",
    "        'https://www.basketball-reference.com/wnba/teams/{}/{}.html'.format(team,year)])\n",
    "        fs = os.path.getsize(target)\n",
    "        if fs < 10:\n",
    "            os.remove(target)\n",
    "            continue\n",
    "    \n",
    "    # load the data\n",
    "    tables[team] = {}\n",
    "\n",
    "    with open(target,'rt', encoding = \"ISO-8859-1\") as fp:\n",
    "        data = fp.read()\n",
    "    tables[team]['name'] = re.findall('{} (.*) Stats'.format(year),data)[0]\n",
    "    bs = BeautifulSoup(data,features=\"lxml\")\n",
    "    #tables[team]['logo'] = re.findall('(http.*png)',str(bs.find_all('img',{\"class\": \"teamlogo\"})[0]))[0]\n",
    "    #tables[team]['conf'] = re.findall('/cbb/conferences/(.*)/{}.html'.format(year),data)[0]\n",
    "\n",
    "    # collect all the tables\n",
    "    m = re.findall(r'<!--[ \\n]*(<div[\\s\\S\\r]+?</div>)[ \\n]*-->',data)\n",
    "    m2 = re.findall(r'(<div class=\"table_outer_container\">[ \\n]*<div class=\"overthrow table_container\" id=\"div_roster\">[\\s\\S\\r]+?</table>[ \\n]*</div>[ \\n]*</div>)',data)\n",
    "    m4 = re.findall(r'(<div class=\"table_outer_container\">[ \\n]*<div class=\"overthrow table_container\" id=\".*\">[\\s\\S\\r]+?</table>[ \\n]*</div>[ \\n]*</div>)',data)\n",
    "\n",
    "    m = m2 + m + m4\n",
    "    print(target,len(m))\n",
    "    for test_table in m:\n",
    "        try:\n",
    "            soup = BeautifulSoup(test_table,features=\"lxml\")\n",
    "            table_id = str(soup.find('table').get('id'))\n",
    "\n",
    "            if table_id in ['wnba_playoffs','team_and_opponent']:\n",
    "                continue\n",
    "            soup.findAll('tr')\n",
    "\n",
    "            table_size = {'on_off':1,'on_off_p':1,'shooting':2,'pbp':1,'playoffs_shooting':2,'playoffs_pbp':1,'contracts':1}\n",
    "\n",
    "            # use getText()to extract the text we need into a list\n",
    "            headers = [th.getText() for th in soup.findAll('tr')[table_size.get(table_id,0)].findAll('th')]\n",
    "\n",
    "            # exclude the first column as we will not need the ranking order from Basketball Reference for the analysis\n",
    "            start_col = 1\n",
    "            if table_id in ['contracts','injury','on_off','on_off_p','roster']:\n",
    "                start_col = 0\n",
    "\n",
    "            headers = headers[start_col:]\n",
    "            rows = soup.findAll('tr')[start_col:]\n",
    "            player_stats = [[td.getText() for td in rows[i].findAll('td')]\n",
    "                        for i in range(len(rows))]\n",
    "\n",
    "            if table_id in ['contracts','injury','on_off','on_off_p','roster']:\n",
    "                player_names = [[td.getText() for td in rows[i].findAll('th')]\n",
    "                            for i in range(len(rows))]\n",
    "                player_stats = [a + b for a,b in zip(player_names[1:],player_stats[1:])]\n",
    "            headers[0] = 'Name'\n",
    "            stats = pd.DataFrame(player_stats, columns = headers).set_index('Name')\n",
    "            if table_id in ['contracts']:\n",
    "                stats = stats.drop(['Player'])\n",
    "                stats = stats.iloc[:stats.index.get_loc('')]\n",
    "\n",
    "            # drop nan\n",
    "            stats = stats[~ stats.index.isin([None])]\n",
    "            # convert to float\n",
    "            obj_cols = stats.loc[:, stats.dtypes == object]\n",
    "            conv_cols = obj_cols.apply(pd.to_numeric, errors = 'ignore')\n",
    "            stats.loc[:, stats.dtypes == object] = conv_cols\n",
    "            \n",
    "            stats = stats.fillna('')\n",
    "            \n",
    "            if True and 'on_off' in table_id:\n",
    "                stats = stats.iloc[~ stats.index.get_loc('Player')]\n",
    "                stats = stats.loc[~ (stats.Split == '')]\n",
    "                stats.index = list(itertools.chain.from_iterable(itertools.repeat(x, 3) for x in [_ for _ in stats.index if _!='']))\n",
    "\n",
    "            #print(table_id,stats.index)\n",
    "            tables[team][table_id]= stats\n",
    "        except KeyboardInterrupt:\n",
    "            raise\n",
    "        except:\n",
    "            #pass\n",
    "            print('FAILED TO PARSE ' +str(soup.find('table').get('id') ))\n",
    "\n",
    "            raise\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open('wnba_{}.pkl'.format(year),'wb') as fp:\n",
    "    pickle.dump(tables,fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tables['LVA'].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}