bwf_ranking_scrape.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f7359a9d",
   "metadata": {},
   "source": [
    "# Web Scrapping BWF Historical World Rankings\n",
    "\n",
    "Building on the from the previous project where I analysed Loh Kean Yew's match performance, I thought it will be interesting to add on ranking as a metric. We could possibly discover some trend when he plays against higher or lower ranked opponents."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ea1ffd2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import requests"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e098d16",
   "metadata": {},
   "source": [
    "The historical rankings can be found on the BWF website at the following [link](https://corporate.bwfbadminton.com/players/historical-rankings/)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "85063ded",
   "metadata": {},
   "outputs": [],
   "source": [
    "base_url = 'https://corporate.bwfbadminton.com/players/historical-rankings/?ryear='\n",
    "endpoint = [str(i) for i in range(2012, 2023)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9785ecd7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2012 : 52\n",
      "2013 : 52\n",
      "2014 : 52\n",
      "2015 : 53\n",
      "2016 : 52\n",
      "2017 : 52\n",
      "2018 : 52\n",
      "2019 : 53\n",
      "2020 : 11\n",
      "2021 : 48\n",
      "2022 : 19\n"
     ]
    }
   ],
   "source": [
    "links = []\n",
    "name = []\n",
    "\n",
    "for year in endpoint:\n",
    "    count = 0\n",
    "    url = base_url + year\n",
    "    response = requests.get(url)\n",
    "    content = response.text\n",
    "    parser = BeautifulSoup(content, 'html.parser')\n",
    "    body = parser.find_all('body')\n",
    "    ranking_container = body[0].find_all('div', class_ = 'ranking-files')\n",
    "    link_container = ranking_container[0].find_all('a')\n",
    "    \n",
    "    for rank in link_container:\n",
    "        links.append(rank.get('href'))\n",
    "        name.append(rank.text.strip() + '.xlsx')\n",
    "        count += 1\n",
    "    print(year,':',count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "eb0415ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "final_rank = list(zip(name, links))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8093feaf",
   "metadata": {},
   "outputs": [],
   "source": [
    "final_rank_df = pd.DataFrame(final_rank, columns = ['rank_week_filename', 'url_link'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "407958d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# correcting one of the error in filename WR 2101-06-01 (Week-22) to WR 2021-06-01 (Week-22)\n",
    "\n",
    "final_rank_df.replace('WR 2101-06-01 (Week-22).xlsx', 'WR 2021-06-01 (Week-22).xlsx', inplace = True)\n",
    "final_rank = [tuple(r) for r in final_rank_df.to_numpy()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c13c7327",
   "metadata": {},
   "outputs": [],
   "source": [
    "final_rank_df.to_csv('filename_url.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5da418f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in final_rank:\n",
    "    response = requests.get(i[1])\n",
    "    output = open(i[0], 'wb')\n",
    "    output.write(response.content)\n",
    "    output.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}