10_ee_extract_s5p_cloud_10km.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "17d55333",
   "metadata": {},
   "outputs": [],
   "source": [
    "import ee\n",
    "import pandas as pd\n",
    "from datetime import datetime, timedelta\n",
    "from dateutil.relativedelta import relativedelta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "8ded554c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# If it's for the first time, authentication is required before running this code\n",
    "# https://developers.google.com/earth-engine/guides/python_install-conda#windows_5\n",
    "ee.Initialize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "bf989627",
   "metadata": {},
   "outputs": [],
   "source": [
    "# earth engine location of the subsets of the 10km grid created separately in the script \"00_create_10_by_10_grid.R\"\n",
    "# 10km grid shapefile needs to be uploaded to assets in your GEE before running this line\n",
    "grid = ee.FeatureCollection(\"projects/ee-akawano/assets/grid_10km\")\n",
    "\n",
    "# google drive associated with your GEE account\n",
    "output_drive = \"s5p_cloud_L3_10km\"\n",
    "timezone = \"utc\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "ec05a0f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://developers.google.com/earth-engine/datasets/catalog/COPERNICUS_S5P_OFFL_L3_CLOUD\n",
    "# 2018-07-04 to 2022-12-14\n",
    "maiac = ee.ImageCollection(\"COPERNICUS/S5P/OFFL/L3_CLOUD\").select(\"cloud_fraction\")\n",
    "maiac_proj = maiac.first().projection()\n",
    "pixel_res = maiac_proj.nominalScale().getInfo()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "5ba3e5b4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['2018-07-01', '2018-10-01', '2019-01-01', '2019-04-01', '2019-07-01', '2019-10-01', '2020-01-01', '2020-04-01', '2020-07-01', '2020-10-01', '2021-01-01', '2021-04-01', '2021-07-01', '2021-10-01', '2022-01-01', '2022-04-01', '2022-07-01']\n",
      "['2018-10-01', '2019-01-01', '2019-04-01', '2019-07-01', '2019-10-01', '2020-01-01', '2020-04-01', '2020-07-01', '2020-10-01', '2021-01-01', '2021-04-01', '2021-07-01', '2021-10-01', '2022-01-01', '2022-04-01', '2022-07-01', '2022-10-01']\n"
     ]
    }
   ],
   "source": [
    "# set start_date and end_date\n",
    "start_date_1 = pd.to_datetime(\"2018-07-01\", format=\"%Y/%m/%d\")\n",
    "end_date_1 = pd.to_datetime(\"2022-08-01\", format=\"%Y/%m/%d\")\n",
    "\n",
    "# Initialize an empty list to store the date strings for the first list\n",
    "start_date_list = []\n",
    "\n",
    "# Loop through the dates for the first list, incrementing by 3 months each time\n",
    "while start_date_1 < end_date_1:\n",
    "    start_date_list.append(start_date_1.strftime(\"%Y-%m-%d\"))\n",
    "    \n",
    "    # Increment the start date by 3 months\n",
    "    start_date_1 += relativedelta(months=3)\n",
    "\n",
    "# Start and end dates for the second list\n",
    "start_date_2 = pd.to_datetime(\"2018-10-01\", format=\"%Y/%m/%d\")\n",
    "end_date_2 = pd.to_datetime(\"2022-11-01\", format=\"%Y/%m/%d\")\n",
    "\n",
    "# Initialize an empty list to store the date strings for the second list\n",
    "end_date_list = []\n",
    "\n",
    "# Loop through the dates for the second list, incrementing by 3 months each time\n",
    "while start_date_2 < end_date_2:\n",
    "    # Format the date as a string in the year-month-day format\n",
    "    end_date_list.append(start_date_2.strftime(\"%Y-%m-%d\"))\n",
    "    \n",
    "    # Increment the start date by 3 months\n",
    "    start_date_2 += relativedelta(months=3)\n",
    "\n",
    "# Print the lists of date strings\n",
    "print(start_date_list)\n",
    "print(end_date_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "0e15f434",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate how many days between start and end days\n",
    "def calculate_length(start_date, end_date):\n",
    "    start_date = pd.to_datetime(start_date)\n",
    "    end_date = pd.to_datetime(end_date)\n",
    "    length_date = (end_date - start_date).days\n",
    "    return length_date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "17d57641",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_image(start_date, length_date):\n",
    "    images_list = ee.List([])\n",
    "    for i in range(0, length_date):\n",
    "        # timezone is default utc in ee.Date.fromYMD\n",
    "        start_date = pd.to_datetime(start_date)\n",
    "        im_date = ee.Date.fromYMD(start_date.year, start_date.month, start_date.day).advance(i, 'day')\n",
    "        im_date_end = im_date.advance(1, 'day')\n",
    "        im = maiac.filterDate(im_date, im_date_end).reduce(ee.Reducer.mean())\n",
    "        im = ee.Image(im).set(\"start_date\", im_date.format(\"yMMdd\"))\n",
    "        images_list = images_list.add(im)\n",
    "    return ee.ImageCollection.fromImages(images_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "c38ddad3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# For each day, calculate average for each cell grid\n",
    "def process_image(start_date, end_date):\n",
    "    length_date = calculate_length(start_date, end_date)\n",
    "    maiac_images = create_image(start_date, length_date)\n",
    "    maiac_images_reduced = maiac_images.map(lambda daily_im: (\n",
    "        daily_im.reduceRegions(\n",
    "            collection = grid,\n",
    "            reducer = ee.Reducer.mean(),\n",
    "            crs = maiac_proj,\n",
    "            scale = pixel_res\n",
    "        ).map(lambda f: f.set(\"start_date\", daily_im.get(\"start_date\")))))\n",
    "    grid_maiac = ee.FeatureCollection(maiac_images_reduced).flatten()\n",
    "    return grid_maiac"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "fd36681c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['grid_id', 'mean', 'start_date']"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# make a list of properties to export \n",
    "export_properties = [\"grid_id\", \"mean\", \"start_date\"]\n",
    "export_properties"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "bf3241c8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saving file as {'cloud_10km_grid_2018-07-01_to_2018-10-01'}\n",
      "saving file as {'cloud_10km_grid_2018-10-01_to_2019-01-01'}\n",
      "saving file as {'cloud_10km_grid_2019-01-01_to_2019-04-01'}\n",
      "saving file as {'cloud_10km_grid_2019-04-01_to_2019-07-01'}\n",
      "saving file as {'cloud_10km_grid_2019-07-01_to_2019-10-01'}\n",
      "saving file as {'cloud_10km_grid_2019-10-01_to_2020-01-01'}\n",
      "saving file as {'cloud_10km_grid_2020-01-01_to_2020-04-01'}\n",
      "saving file as {'cloud_10km_grid_2020-04-01_to_2020-07-01'}\n",
      "saving file as {'cloud_10km_grid_2020-07-01_to_2020-10-01'}\n",
      "saving file as {'cloud_10km_grid_2020-10-01_to_2021-01-01'}\n",
      "saving file as {'cloud_10km_grid_2021-01-01_to_2021-04-01'}\n",
      "saving file as {'cloud_10km_grid_2021-04-01_to_2021-07-01'}\n",
      "saving file as {'cloud_10km_grid_2021-07-01_to_2021-10-01'}\n",
      "saving file as {'cloud_10km_grid_2021-10-01_to_2022-01-01'}\n",
      "saving file as {'cloud_10km_grid_2022-01-01_to_2022-04-01'}\n",
      "saving file as {'cloud_10km_grid_2022-04-01_to_2022-07-01'}\n",
      "saving file as {'cloud_10km_grid_2022-07-01_to_2022-10-01'}\n"
     ]
    }
   ],
   "source": [
    "# loop using start_date_list and end_date_list\n",
    "for start_date, end_date in zip(start_date_list, end_date_list):\n",
    "        grid_cloud = process_image(start_date, end_date)\n",
    "        x = \"cloud_10km_grid\"\n",
    "        y = start_date\n",
    "        t = \"to\"\n",
    "        z = end_date\n",
    "        output_name = \"_\".join([x, y, t, z])\n",
    "        aod_task = ee.batch.Export.table.toDrive(\n",
    "            collection = grid_cloud,\n",
    "            folder = output_drive,\n",
    "            description = output_name,\n",
    "            fileFormat = \"CSV\",\n",
    "            selectors = export_properties)\n",
    "        print(f\"saving file as\", {output_name})\n",
    "        aod_task.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "9a2518a7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saving file as {'cloud_10km_grid_2022-10-01_to_2022-11-01'}\n"
     ]
    }
   ],
   "source": [
    "start_date = '2022-10-01'\n",
    "end_date = '2022-11-01'\n",
    "\n",
    "grid_cloud = process_image(start_date, end_date)\n",
    "x = \"cloud_10km_grid\"\n",
    "y = start_date\n",
    "t = \"to\"\n",
    "z = end_date\n",
    "output_name = \"_\".join([x, y, t, z])\n",
    "aod_task = ee.batch.Export.table.toDrive(\n",
    "    collection = grid_cloud,\n",
    "    folder = output_drive,\n",
    "    description = output_name,\n",
    "    fileFormat = \"CSV\",\n",
    "    selectors = export_properties)\n",
    "print(f\"saving file as\", {output_name})\n",
    "aod_task.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "aab672cc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saving file as {'cloud_10km_grid_2018-07-04_to_2018-10-01'}\n"
     ]
    }
   ],
   "source": [
    "start_date = '2018-07-04'\n",
    "end_date = '2018-10-01'\n",
    "\n",
    "grid_cloud = process_image(start_date, end_date)\n",
    "x = \"cloud_10km_grid\"\n",
    "y = start_date\n",
    "t = \"to\"\n",
    "z = end_date\n",
    "output_name = \"_\".join([x, y, t, z])\n",
    "aod_task = ee.batch.Export.table.toDrive(\n",
    "    collection = grid_cloud,\n",
    "    folder = output_drive,\n",
    "    description = output_name,\n",
    "    fileFormat = \"CSV\",\n",
    "    selectors = export_properties)\n",
    "print(f\"saving file as\", {output_name})\n",
    "aod_task.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "2fc65ad4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saving file as {'cloud_10km_grid_2018-07-04_to_2018-08-01'}\n"
     ]
    }
   ],
   "source": [
    "# Error: cloud_10km_grid_2018-07-04_to_2018-10-01\n",
    "start_date = '2018-07-04'\n",
    "end_date = '2018-08-01'\n",
    "\n",
    "grid_cloud = process_image(start_date, end_date)\n",
    "x = \"cloud_10km_grid\"\n",
    "y = start_date\n",
    "t = \"to\"\n",
    "z = end_date\n",
    "output_name = \"_\".join([x, y, t, z])\n",
    "aod_task = ee.batch.Export.table.toDrive(\n",
    "    collection = grid_cloud,\n",
    "    folder = output_drive,\n",
    "    description = output_name,\n",
    "    fileFormat = \"CSV\",\n",
    "    selectors = export_properties)\n",
    "print(f\"saving file as\", {output_name})\n",
    "aod_task.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "ded5f16c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saving file as {'cloud_10km_grid_2018-08-01_to_2018-09-01'}\n"
     ]
    }
   ],
   "source": [
    "# Error: cloud_10km_grid_2018-07-04_to_2018-10-01\n",
    "start_date = '2018-08-01'\n",
    "end_date = '2018-09-01'\n",
    "\n",
    "grid_cloud = process_image(start_date, end_date)\n",
    "x = \"cloud_10km_grid\"\n",
    "y = start_date\n",
    "t = \"to\"\n",
    "z = end_date\n",
    "output_name = \"_\".join([x, y, t, z])\n",
    "aod_task = ee.batch.Export.table.toDrive(\n",
    "    collection = grid_cloud,\n",
    "    folder = output_drive,\n",
    "    description = output_name,\n",
    "    fileFormat = \"CSV\",\n",
    "    selectors = export_properties)\n",
    "print(f\"saving file as\", {output_name})\n",
    "aod_task.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "63b4392c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saving file as {'cloud_10km_grid_2018-09-01_to_2018-10-01'}\n"
     ]
    }
   ],
   "source": [
    "# Error: cloud_10km_grid_2018-07-04_to_2018-10-01\n",
    "start_date = '2018-09-01'\n",
    "end_date = '2018-10-01'\n",
    "\n",
    "grid_cloud = process_image(start_date, end_date)\n",
    "x = \"cloud_10km_grid\"\n",
    "y = start_date\n",
    "t = \"to\"\n",
    "z = end_date\n",
    "output_name = \"_\".join([x, y, t, z])\n",
    "aod_task = ee.batch.Export.table.toDrive(\n",
    "    collection = grid_cloud,\n",
    "    folder = output_drive,\n",
    "    description = output_name,\n",
    "    fileFormat = \"CSV\",\n",
    "    selectors = export_properties)\n",
    "print(f\"saving file as\", {output_name})\n",
    "aod_task.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "5992f10b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saving file as {'cloud_10km_grid_2018-07-04_to_2018-07-15'}\n"
     ]
    }
   ],
   "source": [
    "# Error cloud_10km_grid_2018-07-04_to_2018-08-01\n",
    "start_date = '2018-07-04'\n",
    "end_date = '2018-07-15'\n",
    "\n",
    "grid_cloud = process_image(start_date, end_date)\n",
    "x = \"cloud_10km_grid\"\n",
    "y = start_date\n",
    "t = \"to\"\n",
    "z = end_date\n",
    "output_name = \"_\".join([x, y, t, z])\n",
    "aod_task = ee.batch.Export.table.toDrive(\n",
    "    collection = grid_cloud,\n",
    "    folder = output_drive,\n",
    "    description = output_name,\n",
    "    fileFormat = \"CSV\",\n",
    "    selectors = export_properties)\n",
    "print(f\"saving file as\", {output_name})\n",
    "aod_task.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "682c47ec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saving file as {'cloud_10km_grid_2018-07-15_to_2018-08-01'}\n"
     ]
    }
   ],
   "source": [
    "# Error cloud_10km_grid_2018-07-04_to_2018-08-01\n",
    "start_date = '2018-07-15'\n",
    "end_date = '2018-08-01'\n",
    "\n",
    "grid_cloud = process_image(start_date, end_date)\n",
    "x = \"cloud_10km_grid\"\n",
    "y = start_date\n",
    "t = \"to\"\n",
    "z = end_date\n",
    "output_name = \"_\".join([x, y, t, z])\n",
    "aod_task = ee.batch.Export.table.toDrive(\n",
    "    collection = grid_cloud,\n",
    "    folder = output_drive,\n",
    "    description = output_name,\n",
    "    fileFormat = \"CSV\",\n",
    "    selectors = export_properties)\n",
    "print(f\"saving file as\", {output_name})\n",
    "aod_task.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "370a7466",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saving file as {'cloud_10km_grid_2018-07-04_to_2018-07-10'}\n"
     ]
    }
   ],
   "source": [
    "# Error cloud_10km_grid_2018-07-04_to_2018-07-15\n",
    "start_date = '2018-07-04'\n",
    "end_date = '2018-07-10'\n",
    "\n",
    "grid_cloud = process_image(start_date, end_date)\n",
    "x = \"cloud_10km_grid\"\n",
    "y = start_date\n",
    "t = \"to\"\n",
    "z = end_date\n",
    "output_name = \"_\".join([x, y, t, z])\n",
    "aod_task = ee.batch.Export.table.toDrive(\n",
    "    collection = grid_cloud,\n",
    "    folder = output_drive,\n",
    "    description = output_name,\n",
    "    fileFormat = \"CSV\",\n",
    "    selectors = export_properties)\n",
    "print(f\"saving file as\", {output_name})\n",
    "aod_task.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "9ceddf14",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saving file as {'cloud_10km_grid_2018-07-10_to_2018-07-15'}\n"
     ]
    }
   ],
   "source": [
    "# Error cloud_10km_grid_2018-07-04_to_2018-07-15\n",
    "start_date = '2018-07-10'\n",
    "end_date = '2018-07-15'\n",
    "\n",
    "grid_cloud = process_image(start_date, end_date)\n",
    "x = \"cloud_10km_grid\"\n",
    "y = start_date\n",
    "t = \"to\"\n",
    "z = end_date\n",
    "output_name = \"_\".join([x, y, t, z])\n",
    "aod_task = ee.batch.Export.table.toDrive(\n",
    "    collection = grid_cloud,\n",
    "    folder = output_drive,\n",
    "    description = output_name,\n",
    "    fileFormat = \"CSV\",\n",
    "    selectors = export_properties)\n",
    "print(f\"saving file as\", {output_name})\n",
    "aod_task.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "540306e6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saving file as {'cloud_10km_grid_2018-07-04_to_2018-07-07'}\n"
     ]
    }
   ],
   "source": [
    "# cloud_10km_grid_2018-07-04_to_2018-07-10\n",
    "start_date = '2018-07-04'\n",
    "end_date = '2018-07-07'\n",
    "\n",
    "grid_cloud = process_image(start_date, end_date)\n",
    "x = \"cloud_10km_grid\"\n",
    "y = start_date\n",
    "t = \"to\"\n",
    "z = end_date\n",
    "output_name = \"_\".join([x, y, t, z])\n",
    "aod_task = ee.batch.Export.table.toDrive(\n",
    "    collection = grid_cloud,\n",
    "    folder = output_drive,\n",
    "    description = output_name,\n",
    "    fileFormat = \"CSV\",\n",
    "    selectors = export_properties)\n",
    "print(f\"saving file as\", {output_name})\n",
    "aod_task.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "05d0a47c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "saving file as {'cloud_10km_grid_2018-07-07_to_2018-07-10'}\n"
     ]
    }
   ],
   "source": [
    "# cloud_10km_grid_2018-07-04_to_2018-07-10\n",
    "start_date = '2018-07-07'\n",
    "end_date = '2018-07-10'\n",
    "\n",
    "grid_cloud = process_image(start_date, end_date)\n",
    "x = \"cloud_10km_grid\"\n",
    "y = start_date\n",
    "t = \"to\"\n",
    "z = end_date\n",
    "output_name = \"_\".join([x, y, t, z])\n",
    "aod_task = ee.batch.Export.table.toDrive(\n",
    "    collection = grid_cloud,\n",
    "    folder = output_drive,\n",
    "    description = output_name,\n",
    "    fileFormat = \"CSV\",\n",
    "    selectors = export_properties)\n",
    "print(f\"saving file as\", {output_name})\n",
    "aod_task.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30289edc",
   "metadata": {},
   "outputs": [],
   "source": [
    "# if you want to cancel tasks:\n",
    "# https://code.earthengine.google.com/tasks"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:ee] *",
   "language": "python",
   "name": "conda-env-ee-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}