notebooks/find-site-panos.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If Dockerfiles have not been modified, connect to the Jupyter server with ```http://localhost:8002/tree?token=find-site-panos```  \n",
    "\n",
    "This notebook describes a pipeline to find all Google Street View panoramas with an available depth map within a bounding rectangle.  \n",
    "The bounding rectangle is specified by providing two [EPSG:4326](https://epsg.io/4326) coordinates that are used as corner points.  \n",
    "Some code and packages are derived from [Street View DEMs](https://github.com/ccheemeng/streetview-dem).  \n",
    "\n",
    "For each valid found panorama, its ID (serving as a primary key), latitude, and longitude are written to a ```.csv``` file.  \n",
    "This ```.csv``` file may be used as an input for [```collect-street-view```](collect-street-view.ipynb).  \n",
    "\n",
    "If necessary, the output ```.csv``` file may be preprocessed with a GIS software like [QGIS](https://www.qgis.org) to further refine the found panoramas beyond being defined by a rectangle."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_dir = \"data\"\n",
    "lat1 = 1.3840874919650237\n",
    "lon1 = 103.73928736838847\n",
    "lat2 = 1.4066928731913269\n",
    "lon2 = 103.76019645180367\n",
    "output_filename = \"yew-tee-locationss.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import aiocsv\n",
    "import aiofiles\n",
    "from aiohttp import ClientSession, ClientTimeout\n",
    "\n",
    "import geo\n",
    "import streetview\n",
    "\n",
    "import asyncio\n",
    "import csv\n",
    "from pathlib import Path\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "async def main(p1, p2, resolution, target_crs, filepath):\n",
    "    Path(filepath).parent.mkdir(parents=True, exist_ok=True)\n",
    "    with open(filepath, 'w', newline='') as csvfile:\n",
    "        csv_writer = csv.writer(csvfile)\n",
    "        csv_writer.writerow([\"id\", \"lat\", \"lon\"])\n",
    "    sample = geo.Sample(p1, p2, resolution)\n",
    "    latlon = list(sample.generate_latlon_samples())\n",
    "    lat = [x[0] for x in latlon]\n",
    "    lon = [x[1] for x in latlon]\n",
    "    radius = sample.search_radius()\n",
    "    await write_panos(lat, lon, radius, target_crs, filepath)\n",
    "\n",
    "async def write_panos(lat, lon, radius, target_crs, filepath):\n",
    "    ids = set()\n",
    "    async with ClientSession(timeout=ClientTimeout(total=None), raise_for_status=True) as session:\n",
    "        async with aiofiles.open(filepath, 'w+') as afp:\n",
    "            writer = aiocsv.AsyncWriter(afp)\n",
    "            await writer.writerow([\"id\", \"lat\", \"lon\"])\n",
    "            for i in range(len(lat)):\n",
    "                async for row in get_latlon(session, ids, lat[i], lon[i], radius, target_crs):\n",
    "                    await writer.writerow(row)\n",
    "\n",
    "async def get_latlon(session, ids, lat, lon, radius, target_crs):\n",
    "    pano = await streetview.StreetViewAPI.find_pano_full(session, lat, lon, radius)\n",
    "    if pano and pano.id not in ids and pano.elevation and pano.depth:\n",
    "        ids.add(pano.id)\n",
    "        yield (pano.id, pano.lat, pano.lon)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cmHafmt9Bn0h0EKGPPLPig: invalid literal for int() with base 2: ''\n",
      "9NIWaw61SMfms-68tWjdsw: invalid literal for int() with base 2: ''\n",
      "TZxyrX9XOnbAGv-t2GZ6dw: invalid literal for int() with base 2: ''\n",
      "3cTDqT6N6o-3i7N6TY1uLg: invalid literal for int() with base 2: ''\n",
      "TZxyrX9XOnbAGv-t2GZ6dw: invalid literal for int() with base 2: ''\n",
      "1Ctmg5mUcI2Kb7TsNOe9Hg: invalid literal for int() with base 2: ''\n",
      "U2A7MYsquNhLQN78ZCtBXA: invalid literal for int() with base 2: ''\n",
      "tyfMjR6f6vvkM6POJ-vUQQ: invalid literal for int() with base 2: ''\n",
      "4_mduq_a3uJZh9Mpm_tuxA: invalid literal for int() with base 2: ''\n",
      "4_mduq_a3uJZh9Mpm_tuxA: invalid literal for int() with base 2: ''\n",
      "tyfMjR6f6vvkM6POJ-vUQQ: invalid literal for int() with base 2: ''\n",
      "kv2gJs1-r22XqV2lblvVFw: invalid literal for int() with base 2: ''\n",
      "TYtkYUVAli1N9o7PTH5Jqg: invalid literal for int() with base 2: ''\n",
      "G7Dp-FLWp7mfowc5pvwIxA: invalid literal for int() with base 2: ''\n",
      "bha1Us-cEwxmjYRkUpmIPA: invalid literal for int() with base 2: ''\n",
      "P46mO74cnnp4_teDa_408g: invalid literal for int() with base 2: ''\n",
      "9s7mzzkEXeQiF-9CL7sPDw: invalid literal for int() with base 2: ''\n",
      "pTyVY_zWmZZ0tORfJTz-Yw: invalid literal for int() with base 2: ''\n",
      "tAPr3vceZTLrCAxUYIYORw: invalid literal for int() with base 2: ''\n",
      "7duuet4dMJPnWrOwihhkuw: invalid literal for int() with base 2: ''\n",
      "7duuet4dMJPnWrOwihhkuw: invalid literal for int() with base 2: ''\n",
      "6Fv_9I-s5hDjpmr-rKXPsA: invalid literal for int() with base 2: ''\n",
      "O-ahqHrfcXFMic_q3-h96A: invalid literal for int() with base 2: ''\n",
      "O-ahqHrfcXFMic_q3-h96A: invalid literal for int() with base 2: ''\n",
      "NtSV-nFmiwjQCzEYTwzaJQ: invalid literal for int() with base 2: ''\n",
      "1-4OT9eKn5CVKFIsWKvC4A: invalid literal for int() with base 2: ''\n",
      "1-4OT9eKn5CVKFIsWKvC4A: invalid literal for int() with base 2: ''\n",
      "IVncfava9AAaklsmsHH1Tg: invalid literal for int() with base 2: ''\n",
      "IVncfava9AAaklsmsHH1Tg: invalid literal for int() with base 2: ''\n",
      "H2C-OuqdfciViIBhELRg2w: invalid literal for int() with base 2: ''\n",
      "Fbm4-8tEBSKue3himq7khQ: invalid literal for int() with base 2: ''\n",
      "Fbm4-8tEBSKue3himq7khQ: invalid literal for int() with base 2: ''\n",
      "H2C-OuqdfciViIBhELRg2w: invalid literal for int() with base 2: ''\n",
      "MT8e8Z_DN0f0rqKV8d9ZwA: invalid literal for int() with base 2: ''\n",
      "KOiNkYoNMgGg0FmnM91pGA: invalid literal for int() with base 2: ''\n",
      "IKS0JENRCwPse6q4Yk1Ryg: invalid literal for int() with base 2: ''\n",
      "K0bjwJaWWyz9PgKDHKQjdQ: index 131336 is out of bounds for axis 0 with size 131336\n",
      "4tP9HhdqhFkahztNJ3S_KQ: invalid literal for int() with base 2: ''\n",
      "K0bjwJaWWyz9PgKDHKQjdQ: index 131336 is out of bounds for axis 0 with size 131336\n",
      "zLroMMHGNLN1tbOwRNfuOA: invalid literal for int() with base 2: ''\n",
      "K0bjwJaWWyz9PgKDHKQjdQ: index 131336 is out of bounds for axis 0 with size 131336\n",
      "valzsWoGRQSHPgoiQ6taZw: invalid literal for int() with base 2: ''\n",
      "valzsWoGRQSHPgoiQ6taZw: invalid literal for int() with base 2: ''\n",
      "pCPTlXhuk4x50qpyAHtt6g: invalid literal for int() with base 2: ''\n",
      "fbJJD78rYYkfGDaBxD66vA: invalid literal for int() with base 2: ''\n",
      "VJusuMJ-7SzJTR_l2b1FHw: invalid literal for int() with base 2: ''\n",
      "jrRpbXtPYoXWq--dmdZAUQ: invalid literal for int() with base 2: ''\n",
      "jrRpbXtPYoXWq--dmdZAUQ: invalid literal for int() with base 2: ''\n",
      "xXhR1p8q2c6sjU1RdV1wTw: invalid literal for int() with base 2: ''\n",
      "djk29Z9y4Dx5Yw9tU9qopw: invalid literal for int() with base 2: ''\n",
      "djk29Z9y4Dx5Yw9tU9qopw: invalid literal for int() with base 2: ''\n",
      "u26nyR3W9PcMnWMULlR17Q: invalid literal for int() with base 2: ''\n",
      "u26nyR3W9PcMnWMULlR17Q: invalid literal for int() with base 2: ''\n",
      "scxQvWQXHoxcYoYBpUbqXQ: invalid literal for int() with base 2: ''\n",
      "97Wc1XunNCo4wHYpzBh3TA: invalid literal for int() with base 2: ''\n",
      "pA_PJD4ys0JTKNcNhe4uYw: invalid literal for int() with base 2: ''\n",
      "mmUDD9aModwMQ7OqHwtgdQ: invalid literal for int() with base 2: ''\n",
      "NSicNJka5OJ5a4j_wzWrtg: invalid literal for int() with base 2: ''\n",
      "X-ywsI0YAk4GxHj24KBOdg: index 131480 is out of bounds for axis 0 with size 131480\n",
      "0u3l0bF4iChECHp5HS_JHg: invalid literal for int() with base 2: ''\n",
      "om1w3WPxWOHToYbCejUAuA: invalid literal for int() with base 2: ''\n",
      "NSicNJka5OJ5a4j_wzWrtg: invalid literal for int() with base 2: ''\n",
      "T4CjFRqOSFIjyRFmT_w5-A: index 131352 is out of bounds for axis 0 with size 131352\n",
      "32fnV4m0gLX3uaK0tKV6iw: invalid literal for int() with base 2: ''\n",
      "4w2I15gihUqrlE9SgTdOTg: invalid literal for int() with base 2: ''\n",
      "T4CjFRqOSFIjyRFmT_w5-A: index 131352 is out of bounds for axis 0 with size 131352\n",
      "UP_dmKJfjU5YGrdOpE7lIA: invalid literal for int() with base 2: ''\n",
      "yAB9Z33EZEqoZ02z24wdXQ: invalid literal for int() with base 2: ''\n",
      "0BHsk2mlLzDmammAXhL2jw: invalid literal for int() with base 2: ''\n",
      "mnLoAngv8xa0Vx5ZnaGZdQ: invalid literal for int() with base 2: ''\n",
      "UP_dmKJfjU5YGrdOpE7lIA: invalid literal for int() with base 2: ''\n",
      "Eiclsz5KIn4G-7kUcA4hdw: invalid literal for int() with base 2: ''\n",
      "Jpi5xpZ9pZpeHDwNsgMm6Q: invalid literal for int() with base 2: ''\n",
      "WTLBw4zdZngushdA7uYLjw: invalid literal for int() with base 2: ''\n",
      "N57cQoZSPNH1WTpSG46MbQ: invalid literal for int() with base 2: ''\n",
      "ld6N5U2sIUeNOn7etHc43w: invalid literal for int() with base 2: ''\n",
      "sZ8_-yeveGs5alJfkjDCZw: invalid literal for int() with base 2: ''\n",
      "lV9Bg3uYpjXREFUU1awYIw: invalid literal for int() with base 2: ''\n",
      "0R7KnsEQ_IM4psy0lV1UPQ: invalid literal for int() with base 2: ''\n",
      "QRDcFOi50AfMClgnPCPbDw: index 131528 is out of bounds for axis 0 with size 131528\n",
      "ov4vGCdkPoRkVEUBuoVZaA: invalid literal for int() with base 2: ''\n",
      "2BbeYL6GMEK4UnOUIrMFgQ: invalid literal for int() with base 2: ''\n",
      "2BbeYL6GMEK4UnOUIrMFgQ: invalid literal for int() with base 2: ''\n",
      "eQDCYVI_2xDAtd6KBM7cpQ: index 131528 is out of bounds for axis 0 with size 131528\n",
      "QRDcFOi50AfMClgnPCPbDw: index 131528 is out of bounds for axis 0 with size 131528\n",
      "ZO4lm7KJRsjwyYjloEaP7Q: invalid literal for int() with base 2: ''\n",
      "X9Bl4et5b7oz9AcHhjWayA: invalid literal for int() with base 2: ''\n",
      "qIJsAI9EqPjQvJ4PQ6Q90Q: invalid literal for int() with base 2: ''\n",
      "czGEZkYn-9VPvPXBLqW6nA: invalid literal for int() with base 2: ''\n",
      "rKeYjiOkjd_PbVY8Uvk-aQ: invalid literal for int() with base 2: ''\n",
      "ZIpZ41FSaEcSAdc21VZkjg: invalid literal for int() with base 2: ''\n",
      "WhpgNjOFwDh5hKn0NIqkWw: invalid literal for int() with base 2: ''\n",
      "XXWR9Bh5OG89iNkiw8Wa4w: invalid literal for int() with base 2: ''\n",
      "WLFE8d8lkBlUmPA5NAMwmQ: invalid literal for int() with base 2: ''\n",
      "CWnet7IU0ov81S9qTJt8Ow: invalid literal for int() with base 2: ''\n",
      "ZFtOwEusAmOgP17PeeUJ7w: invalid literal for int() with base 2: ''\n",
      "efp-pSyFkGBOSwKAKls7zA: invalid literal for int() with base 2: ''\n",
      "yosQAA0jhN2V6ZjxToQeVw: invalid literal for int() with base 2: ''\n",
      "yosQAA0jhN2V6ZjxToQeVw: invalid literal for int() with base 2: ''\n",
      "3kYccW8FOygJgh6BpSr-Iw: invalid literal for int() with base 2: ''\n",
      "3BswIZejN9qBZXVY5nJ4tg: invalid literal for int() with base 2: ''\n",
      "PWW4rt1_gyrF04ALzQ1wHA: invalid literal for int() with base 2: ''\n",
      "ugo0Jn3CJl0gVflMHc6Now: invalid literal for int() with base 2: ''\n",
      "BP-whkWjOamu3sLnQCSnBg: invalid literal for int() with base 2: ''\n",
      "JV99uUHhfDCBxdDRBRQrdA: invalid literal for int() with base 2: ''\n",
      "BP-whkWjOamu3sLnQCSnBg: invalid literal for int() with base 2: ''\n",
      "v_CaK83uBbjj0TTO0pDRGg: invalid literal for int() with base 2: ''\n",
      "evIIE21WYWZl9VojdIOcYw: invalid literal for int() with base 2: ''\n",
      "_EVZKY7qhGpMkkl53h7XlQ: invalid literal for int() with base 2: ''\n",
      "nvlp_crY5E-sZeivr_T3aA: invalid literal for int() with base 2: ''\n",
      "i0Z9bchDwpytatXFb5yw1A: invalid literal for int() with base 2: ''\n",
      "v_CaK83uBbjj0TTO0pDRGg: invalid literal for int() with base 2: ''\n",
      "evIIE21WYWZl9VojdIOcYw: invalid literal for int() with base 2: ''\n",
      "ikGqYQ2hyRfiZ0c1j_7vtg: invalid literal for int() with base 2: ''\n",
      "0epc9_g4Y6Wi7OBJwFGySQ: invalid literal for int() with base 2: ''\n",
      "ZBvBnFGEaNyoe06pvWhrQQ: invalid literal for int() with base 2: ''\n",
      "jnujmYlgIZaXyJfMsIyWAw: invalid literal for int() with base 2: ''\n",
      "jnujmYlgIZaXyJfMsIyWAw: invalid literal for int() with base 2: ''\n",
      "60dJ9de84CgHHLpSecrHoQ: invalid literal for int() with base 2: ''\n",
      "2HKs5X1t-jyGg3TZzx3X0A: invalid literal for int() with base 2: ''\n",
      "RIGCwI0b5mID9jvvppK2Xg: invalid literal for int() with base 2: ''\n",
      "08hB8GEjs0X97VxWlHj0Bg: invalid literal for int() with base 2: ''\n",
      "08hB8GEjs0X97VxWlHj0Bg: invalid literal for int() with base 2: ''\n"
     ]
    }
   ],
   "source": [
    "await main((lat1, lon1), (lat2, lon2), 10, 4326, os.path.join(target_dir, output_filename))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}