Notebook 2. webscrape images of attractions.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import src.parse_funcs as parse\n",
    "import csv\n",
    "import sys\n",
    "reload(sys)\n",
    "sys.setdefaultencoding('utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>country</th>\n",
       "      <th>attraction</th>\n",
       "      <th>url</th>\n",
       "      <th>last_page_number</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Australia</td>\n",
       "      <td>Sydney_Harbour-Sydney_New_South_Wales</td>\n",
       "      <td>http://www.tripadvisor.com/Attraction_Review-g...</td>\n",
       "      <td>1520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Australia</td>\n",
       "      <td>Australian_War_Memorial-Canberra_Australian_Ca...</td>\n",
       "      <td>http://www.tripadvisor.com/Attraction_Review-g...</td>\n",
       "      <td>751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Australia</td>\n",
       "      <td>Bondi_to_Coogee_Beach_Coastal_Walk-Sydney_New_...</td>\n",
       "      <td>http://www.tripadvisor.com/Attraction_Review-g...</td>\n",
       "      <td>955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Australia</td>\n",
       "      <td>Sydney_Opera_House-Sydney_New_South_Wales</td>\n",
       "      <td>http://www.tripadvisor.com/Attraction_Review-g...</td>\n",
       "      <td>1340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Australia</td>\n",
       "      <td>Shrine_of_Remembrance-Melbourne_Victoria</td>\n",
       "      <td>http://www.tripadvisor.com/Attraction_Review-g...</td>\n",
       "      <td>408</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     country                                         attraction  \\\n",
       "0  Australia              Sydney_Harbour-Sydney_New_South_Wales   \n",
       "1  Australia  Australian_War_Memorial-Canberra_Australian_Ca...   \n",
       "2  Australia  Bondi_to_Coogee_Beach_Coastal_Walk-Sydney_New_...   \n",
       "3  Australia          Sydney_Opera_House-Sydney_New_South_Wales   \n",
       "4  Australia           Shrine_of_Remembrance-Melbourne_Victoria   \n",
       "\n",
       "                                                 url  last_page_number  \n",
       "0  http://www.tripadvisor.com/Attraction_Review-g...              1520  \n",
       "1  http://www.tripadvisor.com/Attraction_Review-g...               751  \n",
       "2  http://www.tripadvisor.com/Attraction_Review-g...               955  \n",
       "3  http://www.tripadvisor.com/Attraction_Review-g...              1340  \n",
       "4  http://www.tripadvisor.com/Attraction_Review-g...               408  "
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('data/attractions.csv')\n",
    "df[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "url= df[df['country'] == \"Australia\"].url[4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "r,soup = parse.soupify(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "address = soup.find_all('div', 'detail_section address')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'St. Kilda Road, Melbourne, Victoria 3001, Australia'"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "address[0].get_text()\n",
    "#r[0].string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def address_extract(url):\n",
    "    r,soup = parse.soupify(url)\n",
    "    address = soup.find_all('div', 'detail_section address')\n",
    "    return address[0].get_text()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "list_of_urls = df.url\n",
    "list_of_address = []\n",
    "for x in list_of_urls:\n",
    "    list_of_address.append(address_extract(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "location_data = zip(df.country, df.attraction, list_of_address)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/location.csv','wb') as out:\n",
    "    csv_out=csv.writer(out)\n",
    "    csv_out.writerow(['country','attraction','address'])\n",
    "    for row in location_data:\n",
    "        csv_out.writerow(row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting Images for each location"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_img(soup):\n",
    "    image=[]\n",
    "    images = soup.find_all('div','prw_rup prw_common_centered_image photo')\n",
    "    links = []\n",
    "    for x in images:\n",
    "        image.extend(x.find_all('img', 'centeredImg noscript'))\n",
    "\n",
    "    for x in image:\n",
    "        links.append(str(x.get('src')))\n",
    "    return links"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "image = get_img(soup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://media-cdn.tripadvisor.com/media/photo-s/0d/43/95/3a/caption.jpg',\n",
       " 'https://media-cdn.tripadvisor.com/media/photo-s/0d/43/95/4a/caption.jpg',\n",
       " 'https://media-cdn.tripadvisor.com/media/photo-s/0d/43/95/78/caption.jpg',\n",
       " 'https://media-cdn.tripadvisor.com/media/photo-w/10/d6/26/35/okavango-house-boats.jpg',\n",
       " 'https://media-cdn.tripadvisor.com/media/photo-s/10/d6/26/60/okavango-house-boats.jpg',\n",
       " 'https://media-cdn.tripadvisor.com/media/photo-w/10/d6/26/88/okavango-house-boats.jpg',\n",
       " 'https://media-cdn.tripadvisor.com/media/photo-s/10/d6/26/ab/okavango-house-boats.jpg',\n",
       " 'https://media-cdn.tripadvisor.com/media/photo-w/10/f1/cb/79/photo2jpg.jpg',\n",
       " 'https://media-cdn.tripadvisor.com/media/photo-s/10/f1/cb/7a/photo3jpg.jpg']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Automating for all URLs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n",
      "20\n",
      "30\n",
      "40\n",
      "50\n",
      "60\n",
      "70\n",
      "80\n",
      "90\n",
      "100\n",
      "110\n",
      "120\n",
      "130\n",
      "140\n",
      "150\n",
      "160\n",
      "170\n",
      "180\n",
      "190\n",
      "200\n",
      "210\n",
      "220\n",
      "230\n",
      "240\n",
      "250\n",
      "260\n",
      "270\n",
      "280\n",
      "290\n",
      "300\n",
      "310\n",
      "320\n",
      "330\n",
      "340\n",
      "350\n",
      "360\n",
      "370\n",
      "380\n",
      "390\n",
      "400\n",
      "410\n",
      "420\n",
      "430\n",
      "440\n",
      "450\n",
      "460\n",
      "470\n",
      "480\n",
      "490\n",
      "500\n",
      "510\n",
      "520\n",
      "530\n",
      "540\n",
      "550\n",
      "560\n",
      "570\n",
      "580\n",
      "590\n",
      "600\n",
      "610\n",
      "620\n",
      "630\n",
      "640\n",
      "650\n",
      "660\n",
      "670\n",
      "680\n",
      "690\n",
      "700\n",
      "710\n",
      "720\n",
      "730\n",
      "740\n",
      "750\n",
      "760\n",
      "770\n",
      "780\n",
      "790\n",
      "800\n",
      "810\n",
      "820\n",
      "830\n",
      "840\n",
      "850\n",
      "860\n",
      "870\n",
      "880\n",
      "890\n",
      "900\n",
      "910\n",
      "920\n",
      "930\n",
      "940\n",
      "950\n",
      "960\n",
      "970\n",
      "980\n",
      "990\n",
      "1000\n",
      "1010\n",
      "1020\n",
      "1030\n",
      "1040\n",
      "1050\n",
      "1060\n",
      "1070\n",
      "1080\n",
      "1090\n",
      "1100\n",
      "1110\n",
      "1120\n",
      "1130\n",
      "1140\n",
      "1150\n",
      "1160\n",
      "1170\n",
      "1180\n",
      "1190\n",
      "1200\n",
      "1210\n",
      "1220\n",
      "1230\n",
      "1240\n",
      "1250\n",
      "1260\n",
      "1270\n",
      "1280\n",
      "1290\n",
      "1300\n",
      "1310\n",
      "1320\n",
      "1330\n",
      "1340\n",
      "1350\n",
      "1360\n",
      "1370\n",
      "1380\n",
      "1390\n",
      "1400\n",
      "1410\n",
      "1420\n",
      "1430\n",
      "1440\n",
      "1450\n",
      "1460\n",
      "1470\n",
      "1480\n",
      "1490\n",
      "1500\n",
      "1510\n",
      "1520\n",
      "1530\n",
      "1540\n",
      "1550\n",
      "1560\n",
      "1570\n",
      "1580\n",
      "1590\n",
      "1600\n",
      "1610\n",
      "1620\n",
      "1630\n",
      "1640\n",
      "1650\n",
      "1660\n",
      "1670\n",
      "1680\n",
      "1690\n",
      "1700\n",
      "1710\n",
      "1720\n",
      "1730\n",
      "1740\n",
      "1750\n",
      "1760\n",
      "1770\n",
      "1780\n",
      "1790\n",
      "1800\n",
      "1810\n",
      "1820\n",
      "1830\n",
      "1840\n",
      "1850\n",
      "1860\n",
      "1870\n",
      "1880\n",
      "1890\n",
      "1900\n",
      "1910\n",
      "1920\n",
      "1930\n",
      "1940\n",
      "1950\n",
      "1960\n",
      "1970\n",
      "1980\n",
      "1990\n",
      "2000\n",
      "2010\n",
      "2020\n",
      "2030\n",
      "2040\n",
      "2050\n",
      "2060\n",
      "2070\n",
      "2080\n",
      "2090\n",
      "2100\n",
      "2110\n",
      "2120\n",
      "2130\n",
      "2140\n",
      "2150\n",
      "2160\n",
      "2170\n",
      "2180\n",
      "2190\n",
      "2200\n",
      "2210\n",
      "2220\n",
      "2230\n",
      "2240\n",
      "2250\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "\n",
    "images_total = []\n",
    "\n",
    "for x in df.url:\n",
    "    r , soup = parse.soupify(x)\n",
    "    image = get_img(soup)\n",
    "    i+=1\n",
    "    if i%10 == 0:\n",
    "        print i\n",
    "    images_total.append(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "i_len = map(len, images_total)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "attraction =[]\n",
    "for i in range(len(df.attraction)):\n",
    "    attraction.append([df.attraction[i]]*i_len[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "attraction_label = []\n",
    "for x in attraction:\n",
    "    attraction_label +=x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "img_links = []\n",
    "for x in images_total:\n",
    "    img_links+=x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "imgs = zip(attraction_label, img_links)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/images.csv','wb') as out:\n",
    "    csv_out=csv.writer(out)\n",
    "    csv_out.writerow(['attraction','image_url'])\n",
    "    for row in imgs:\n",
    "        csv_out.writerow(row)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}