Skip to content

Commit

Permalink
catch ConnectionError and more general exceptions in the scraper
Browse files Browse the repository at this point in the history
  • Loading branch information
kronicka committed Jan 3, 2019
1 parent 4c2fc6e commit 20f31b9
Showing 1 changed file with 22 additions and 8 deletions.
30 changes: 22 additions & 8 deletions scraper.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,28 @@ def scrape_life_expectancy(country: str) -> int:
widget_url = 'https://databank.worldbank.org/data/views/reports/reportwidget.aspx'
report_url = f'?Report_Name=CountryProfile&Id=b450fd57&tbar=y&dd=y&inf=n&zm=n&country={country}'
data_url = widget_url + report_url

response = requests.get(data_url)

onclick_value = "loadMetaData('SP.DYN.LE00.IN','S','Series','Life expectancy at birth, total (years)','2','1801')"
life_expectancy_years = BeautifulSoup(response.text, 'html.parser') \
.find(attrs={'onclick': onclick_value}) \
.parent \
.find_all('td')[-1] \
.find('div').get_text()
life_expectancy_years = None

try:
response = requests.get(data_url)
life_expectancy_years = BeautifulSoup(response.text, 'html.parser') \
.find(attrs={'onclick': onclick_value}) \
.parent \
.find_all('td')[-1] \
.find('div').get_text()
except requests.exceptions.ConnectionError:
print('Couldn\'t reach the World Bank server, pal, fix your connection for the most recent data.')
print('Using locally stored 2016 data for countries instead.')
except requests.exceptions.RequestException as e:
print(e)

return life_expectancy_years


def store_scraped_data_in_file():
"""
Evoke this function to manually store the scraped data locally, for cases when there's no Internet connection.
The dict with key-value (country-life_expectancy) pairs will be stored in countries.txt
"""
pass

0 comments on commit 20f31b9

Please sign in to comment.