Zillow API Search Results.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get Search Results Using Zillow API \n",
    "\n",
    "### Overview\n",
    "\n",
    "| Detail | Information | \n",
    "| --- | --- | \n",
    "| Input Data Source | Zillow API through Rapid API |\n",
    "| Output Data Source | Pandas DataFrame |\n",
    "| Future Developments | Output to DB, Database Modelling & Web Application|\n",
    "| External References | Rapid API, Zillow API, and Ariel Herrera |\n",
    "\n",
    "### History\n",
    "| Date | Details | \n",
    "| --- | --- | \n",
    "| 24 April 2021 | Search results from Zillow API |"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load libraries\n",
    "import pandas as pd \n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# show all cols in display\n",
    "pd.options.display.max_columns = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "api = pd.read_csv('.../API.csv')\n",
    "zwsid = api.loc[api['API'] == 'zillow']['KEY'].iloc[0]\n",
    "rapid_api_key = api.loc[api['API'] == 'rapid']['KEY'].iloc[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Determine street, city, state and zipcode variable:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# address API parameters\n",
    "street = '577 E 13th Ave'\n",
    "city = 'Salt Lake City'\n",
    "state = 'UT'\n",
    "zipcode = '84103'\n",
    "\n",
    "# street_formatted = street.replace(\" \", \"%20\") #URL space encoding\n",
    "# street_formatted\n",
    "\n",
    "# city_formatted = city.replace(\" \", \"%20\") \n",
    "# city_formatted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def search_results(zwsid, \n",
    "                   rapid_api_key,\n",
    "                   street,\n",
    "                   city,\n",
    "                   state,\n",
    "                   zipcode,\n",
    "                   zestimate = True,\n",
    "                   rent_zestimate = True):\n",
    "    \"\"\"\n",
    "    Find a property for a specified address. \n",
    "    \n",
    "    The content results contains the address for the property, zillow property id (ZPID) \n",
    "    and current Zestimate, date when Zestimate was computed, a valuation range, Zestimate\n",
    "    ranking for the property within its zipcode. \n",
    "    \n",
    "    The GetSearchResults API Web Service is located at: \n",
    "    https://www.zillow.com/howto/api/GetSearchResults.htm\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    @zwsid [string]: Zillow web service identified to access data from Zillow API\n",
    "    @rapid_api_key [string]: Key to access data from Rapid API\n",
    "    @street [string]: The street of the property\n",
    "    @city [string]: The city of the property\n",
    "    @state [string]: The state of the property\n",
    "    @zipcode [string]: The zipcode of the property\n",
    "    @zestimate [boolean]: Return Rent Zestimate information if available\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    [json] API response\n",
    "    \"\"\"\n",
    "    # adjust params for paylod\n",
    "    street_formatted = street.replace(\" \", \"%20\") #URL space encoding\n",
    "    city_formatted = city.replace(\" \", \"%20\") # URL space encoding \n",
    "    if zestimate == True:\n",
    "        zestimate_str = \"true\"\n",
    "    else:\n",
    "        zestimate_str = \"false\"\n",
    "    if rent_zestimate == True:\n",
    "        rent_zestimate_str = \"true\"\n",
    "    else:\n",
    "        rent_zestimate_str = \"false\"\n",
    "    \n",
    "    # url for api \n",
    "    url = \"https://zillowdimashirokovv1.p.rapidapi.com/GetSearchResults.htm\"\n",
    "\n",
    "    # payload to send parameters \n",
    "    payload = \"citystatezip={1}%20{2}%20{3}&zws-id={0}&rentzestimate=%5B{5}%2C{6}%5D&address={4}\".format(zwsid,\n",
    "                       city_formatted,\n",
    "                       state,\n",
    "                       str(zipcode),\n",
    "                       street_formatted,\n",
    "                       zestimate_str,\n",
    "                       rent_zestimate_str)\n",
    "    \n",
    "    # header\n",
    "    headers = {\n",
    "    'content-type': \"application/x-www-form-urlencoded\",\n",
    "    'x-rapidapi-key': rapid_api_key,\n",
    "    'x-rapidapi-host': \"ZillowdimashirokovV1.p.rapidapi.com\"\n",
    "    }\n",
    "\n",
    "    # response\n",
    "    response = requests.request(\"POST\", url, data=payload, headers=headers)\n",
    "\n",
    "    # check if there's an error on the request\n",
    "    if response.status_code == 200: # no error\n",
    "        return response.text\n",
    "    else:\n",
    "        return \"\"\"Error {0}.See API source documentation for message and error\n",
    "        code - https://www.zillow.com/howto/api/GetSearchResults.htm\n",
    "        \"\"\".format(response.status_code)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_search_results(response_content):\n",
    "    \"\"\"\n",
    "    Parse search results to tabular format\n",
    "    \n",
    "    Parameters\n",
    "    ----------\n",
    "    @response_content [string]: Response text from GetSearchResults API\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    [dataframe] Tabular response\n",
    "    \"\"\"\n",
    "    soup = BeautifulSoup(response_content, \"xml\")\n",
    "    \n",
    "    content_dict = {\n",
    "        \"street\": [soup.find('street').text],\n",
    "        \"city\": [soup.find('city').text],\n",
    "        \"state\": [soup.find('state').text],\n",
    "        \"zipcode\": [soup.find('zipcode').text],\n",
    "        \"zpid\": [soup.find('zpid').text],\n",
    "        \"home_details\": [soup.find('homedetails').text],\n",
    "        #\"graphsanddata\": [soup.find('graphsanddata').text],\n",
    "        \"mapthishome\": [soup.find('mapthishome').text],\n",
    "        \"comparables\": [soup.find('comparables').text],\n",
    "        \"latitude\": [soup.find(\"last-updated\").text],\n",
    "        \"longitude\": [soup.find('valueChange').text], \n",
    "        \"zestimate_amount\": [soup.find('amount').text],\n",
    "        \"zestimate_last_updated\": [soup.find('last-updated').text],\n",
    "        \"zestimate_value_change\": [soup.find('valueChange').text],\n",
    "        \"zestimate_range_low\": [soup.find('valuationRange').find('low').text],\n",
    "        \"zestimate_range_high\": [soup.find('valuationRange').find('high').text],\n",
    "        \"zillow_index_value\": [soup.find('zindexValue').text]\n",
    "        #\"zillow_index_1YChange\": [soup.find('zindexOneYearChange').text]\n",
    "    }\n",
    "    \n",
    "    return pd.DataFrame.from_dict(content_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get api response\n",
    "search_results_response = search_results(zwsid=zwsid,\n",
    "                                rapid_api_key=rapid_api_key, \n",
    "                                street=street, \n",
    "                                city=city, \n",
    "                                state=state, \n",
    "                                zipcode=zipcode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>street</th>\n",
       "      <th>city</th>\n",
       "      <th>state</th>\n",
       "      <th>zipcode</th>\n",
       "      <th>zpid</th>\n",
       "      <th>home_details</th>\n",
       "      <th>mapthishome</th>\n",
       "      <th>comparables</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>zestimate_amount</th>\n",
       "      <th>zestimate_last_updated</th>\n",
       "      <th>zestimate_value_change</th>\n",
       "      <th>zestimate_range_low</th>\n",
       "      <th>zestimate_range_high</th>\n",
       "      <th>zillow_index_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>577 E 13th Ave</td>\n",
       "      <td>Salt Lake City</td>\n",
       "      <td>UT</td>\n",
       "      <td>84103</td>\n",
       "      <td>12717797</td>\n",
       "      <td>https://www.zillow.com/homedetails/577-E-13th-...</td>\n",
       "      <td>http://www.zillow.com/homes/12717797_zpid/</td>\n",
       "      <td>http://www.zillow.com/homes/comps/12717797_zpid/</td>\n",
       "      <td>04/24/2021</td>\n",
       "      <td>40502</td>\n",
       "      <td>1007069</td>\n",
       "      <td>04/24/2021</td>\n",
       "      <td>40502</td>\n",
       "      <td>926503</td>\n",
       "      <td>1097705</td>\n",
       "      <td>418,700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           street            city state zipcode      zpid  \\\n",
       "0  577 E 13th Ave  Salt Lake City    UT   84103  12717797   \n",
       "\n",
       "                                        home_details  \\\n",
       "0  https://www.zillow.com/homedetails/577-E-13th-...   \n",
       "\n",
       "                                  mapthishome  \\\n",
       "0  http://www.zillow.com/homes/12717797_zpid/   \n",
       "\n",
       "                                        comparables    latitude longitude  \\\n",
       "0  http://www.zillow.com/homes/comps/12717797_zpid/  04/24/2021     40502   \n",
       "\n",
       "  zestimate_amount zestimate_last_updated zestimate_value_change  \\\n",
       "0          1007069             04/24/2021                  40502   \n",
       "\n",
       "  zestimate_range_low zestimate_range_high zillow_index_value  \n",
       "0              926503              1097705            418,700  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# transform xml response to df\n",
    "df = parse_search_results(response_content = search_results_response)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1007069'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get zestimate \n",
    "zestimate_value = df['zestimate_amount'].iloc[0]\n",
    "zestimate_value"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get search results for multiple listings: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "addr = pd.read_csv('AddressParameters.csv', quotechar=\"'\")\n",
    "addr = addr.astype(str)\n",
    "addr['street'] = addr['street'].replace(\" \", \"%20\", regex=True)\n",
    "addr['city'] = addr['city'].replace(\" \", \"%20\", regex=True)\n",
    "# print(addr.iloc[0][0])\n",
    "# print(addr.iloc[0][1])\n",
    "# print(addr.iloc[0][2])\n",
    "# print(addr.iloc[0][3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "multiple_addr = []\n",
    "for index, row in addr.iterrows():\n",
    "    #print(row['street'], row['city'])\n",
    "    search_results_response = search_results(zwsid=zwsid,\n",
    "                                rapid_api_key=rapid_api_key, \n",
    "                                street=row['street'], \n",
    "                                city=row['city'], \n",
    "                                state=row['state'], \n",
    "                                zipcode=row['zipcode'])\n",
    "\n",
    "    multiple_addr.append(search_results_response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = []\n",
    "for x in range(len(multiple_addr)):\n",
    "    #print(test[x])\n",
    "    df = parse_search_results(response_content = multiple_addr[x])\n",
    "    #print(df)\n",
    "    result.append(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>street</th>\n",
       "      <th>city</th>\n",
       "      <th>state</th>\n",
       "      <th>zipcode</th>\n",
       "      <th>zpid</th>\n",
       "      <th>home_details</th>\n",
       "      <th>mapthishome</th>\n",
       "      <th>comparables</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>zestimate_amount</th>\n",
       "      <th>zestimate_last_updated</th>\n",
       "      <th>zestimate_value_change</th>\n",
       "      <th>zestimate_range_low</th>\n",
       "      <th>zestimate_range_high</th>\n",
       "      <th>zillow_index_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>25 N 1000 W</td>\n",
       "      <td>Salt Lake City</td>\n",
       "      <td>UT</td>\n",
       "      <td>84116</td>\n",
       "      <td>12715445</td>\n",
       "      <td>https://www.zillow.com/homedetails/25-N-1000-W...</td>\n",
       "      <td>http://www.zillow.com/homes/12715445_zpid/</td>\n",
       "      <td>http://www.zillow.com/homes/comps/12715445_zpid/</td>\n",
       "      <td>04/24/2021</td>\n",
       "      <td>14896</td>\n",
       "      <td>280171</td>\n",
       "      <td>04/24/2021</td>\n",
       "      <td>14896</td>\n",
       "      <td>198921</td>\n",
       "      <td>310990</td>\n",
       "      <td>158,700</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        street            city state zipcode      zpid  \\\n",
       "0  25 N 1000 W  Salt Lake City    UT   84116  12715445   \n",
       "\n",
       "                                        home_details  \\\n",
       "0  https://www.zillow.com/homedetails/25-N-1000-W...   \n",
       "\n",
       "                                  mapthishome  \\\n",
       "0  http://www.zillow.com/homes/12715445_zpid/   \n",
       "\n",
       "                                        comparables    latitude longitude  \\\n",
       "0  http://www.zillow.com/homes/comps/12715445_zpid/  04/24/2021     14896   \n",
       "\n",
       "  zestimate_amount zestimate_last_updated zestimate_value_change  \\\n",
       "0           280171             04/24/2021                  14896   \n",
       "\n",
       "  zestimate_range_low zestimate_range_high zillow_index_value  \n",
       "0              198921               310990            158,700  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "dff = pd.concat(result, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               street            city state zipcode       zpid  \\\n",
      "0  1023 W Learned Ave  Salt Lake City    UT   84116   12715444   \n",
      "1         25 N 1000 W  Salt Lake City    UT   84116   12715445   \n",
      "2         23 N 1000 W  Salt Lake City    UT   84116   12715446   \n",
      "3         435 N 900 W  Salt Lake City    UT   84116   12715198   \n",
      "4   821 W Fremont Ave  Salt lake city    UT   84104  305057101   \n",
      "\n",
      "                                        home_details  \\\n",
      "0  https://www.zillow.com/homedetails/1023-W-Lear...   \n",
      "1  https://www.zillow.com/homedetails/25-N-1000-W...   \n",
      "2  https://www.zillow.com/homedetails/23-N-1000-W...   \n",
      "3  https://www.zillow.com/homedetails/435-N-900-W...   \n",
      "4  https://www.zillow.com/homedetails/821-W-Fremo...   \n",
      "\n",
      "                                   mapthishome  \\\n",
      "0   http://www.zillow.com/homes/12715444_zpid/   \n",
      "1   http://www.zillow.com/homes/12715445_zpid/   \n",
      "2   http://www.zillow.com/homes/12715446_zpid/   \n",
      "3   http://www.zillow.com/homes/12715198_zpid/   \n",
      "4  http://www.zillow.com/homes/305057101_zpid/   \n",
      "\n",
      "                                         comparables    latitude longitude  \\\n",
      "0   http://www.zillow.com/homes/comps/12715444_zpid/  04/24/2021     16233   \n",
      "1   http://www.zillow.com/homes/comps/12715445_zpid/  04/24/2021     14896   \n",
      "2   http://www.zillow.com/homes/comps/12715446_zpid/  04/24/2021     15342   \n",
      "3   http://www.zillow.com/homes/comps/12715198_zpid/  04/24/2021     25350   \n",
      "4  http://www.zillow.com/homes/comps/305057101_zpid/  01/01/1970             \n",
      "\n",
      "  zestimate_amount zestimate_last_updated zestimate_value_change  \\\n",
      "0           417435             04/24/2021                  16233   \n",
      "1           280171             04/24/2021                  14896   \n",
      "2           306891             04/24/2021                  15342   \n",
      "3           331128             04/24/2021                  25350   \n",
      "4                              01/01/1970                          \n",
      "\n",
      "  zestimate_range_low zestimate_range_high zillow_index_value  \n",
      "0              379866               455004            158,700  \n",
      "1              198921               310990            158,700  \n",
      "2              273133               340649            158,700  \n",
      "3              294704               364241            166,300  \n",
      "4                                                     165,900  \n"
     ]
    }
   ],
   "source": [
    "print(dff.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# END Notebook"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}