diff --git a/docs/user_guide/gallery/met_ascii2nc.ipynb b/docs/user_guide/gallery/met_ascii2nc.ipynb new file mode 100644 index 0000000..b3aa573 --- /dev/null +++ b/docs/user_guide/gallery/met_ascii2nc.ipynb @@ -0,0 +1,326 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 🌐 MET ASCII2NC \n", + "\n", + "> Demonstrates the following:\n", + ">\n", + "> - Get a timeseries of data\n", + "> - Modify and writing DataFrame contents using Polars to a specific format required by MET.\n", + "\n", + "The Model Evaluation Tools (MET) software is used by the NOAA, NRL, BoM, and other meteorological institutions to verify numerical weather prediction forecasts. You can use your own data to verify forecast grids, but you need to convert your data to an ASCII file. Then, MET's [ASCII2NC](https://met.readthedocs.io/en/latest/Users_Guide/reformat_point.html#ascii2nc-tool) tool can convert that file into a NetCDF that MET can read in.\n", + "\n", + "This notebook demonstrates how to use Polars to convert a SynopticPy DataFrame to a file that MET can read.\n", + "\n", + "> WARNING: I haven't actually tested that the file it writes can be used by MET's ASCII2NC tool. This is primarily a proof of concept. Please open a PR if you want to see this feature improved and tested." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "import synoptic\n", + "import polars as pl" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "SynopticPy comes with an easy function to write a DataFrame to MET's ASCII2NC format for you" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "🚚💨 Speedy delivery from Synoptic's \u001b[32mtimeseries\u001b[0m service.\n", + "📦 Received data from \u001b[36m1\u001b[0m stations (0.31 seconds).\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/blaylock/GITHUB/SynopticPy/src/synoptic/polars_namespace.py:37: UserWarning: `write_met` is experimental and proof of concept. NEEDS TESTING WIT MET's ASCII2NC tool.\n", + " warnings.warn(\n" + ] + } + ], + "source": [ + "df = synoptic.TimeSeries(stid=\"ubkbk,wbb\", recent=60).df()\n", + "df.synoptic.write_met(\"sample_ascii_for_met.txt\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now let's look at what that file we wrote looks like..." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "MESONET WBB 20241222_033100 40.76623 -111.84755 1464.8688 TMP NA NA passed 1.978\n", + "MESONET WBB 20241222_033200 40.76623 -111.84755 1464.8688 TMP NA NA passed 1.972\n", + "MESONET WBB 20241222_033300 40.76623 -111.84755 1464.8688 TMP NA NA passed 2.028\n", + "MESONET WBB 20241222_033400 40.76623 -111.84755 1464.8688 TMP NA NA passed 2.139\n", + "MESONET WBB 20241222_033500 40.76623 -111.84755 1464.8688 TMP NA NA passed 2.222\n", + "MESONET WBB 20241222_033600 40.76623 -111.84755 1464.8688 TMP NA NA passed 2.283\n", + "MESONET WBB 20241222_033700 40.76623 -111.84755 1464.8688 TMP NA NA passed 2.289\n", + "MESONET WBB 20241222_033800 40.76623 -111.84755 1464.8688 TMP NA NA passed 2.344\n", + "MESONET WBB 20241222_033900 40.76623 -111.84755 1464.8688 TMP NA NA passed 2.361\n", + "MESONET WBB 20241222_034000 40.76623 -111.84755 1464.8688 TMP NA NA passed 2.333\n" + ] + } + ], + "source": [ + "%%bash\n", + "head sample_ascii_for_met.txt" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "That's nice, but you might wonder how it works. You can look at the source code, or I'll just show you below the most important parts..." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "🚚💨 Speedy delivery from Synoptic's \u001b[32mtimeseries\u001b[0m service.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "📦 Received data from \u001b[36m1\u001b[0m stations (0.22 seconds).\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "shape: (1_155, 23)
stiddate_timevariablesensor_indexis_derivedvaluevalue_stingunitsidnameelevationlatitudelongitudemnet_idstatetimezoneelev_demperiod_of_record_startperiod_of_record_endqc_flaggedis_restrictedrestricted_metadatais_active
strdatetime[μs, UTC]stru32boolf64strstru32strf64f64f64u32strstrf64datetime[μs, UTC]datetime[μs, UTC]boolboolboolbool
"WBB"2024-12-22 03:31:00 UTC"air_temp"1false1.978null"Celsius"1"U of U William Browning Buildi…4806.040.76623-111.84755153"UT""America/Denver"4727.71997-01-01 00:00:00 UTC2024-12-22 04:05:00 UTCfalsefalsefalsetrue
"WBB"2024-12-22 03:32:00 UTC"air_temp"1false1.972null"Celsius"1"U of U William Browning Buildi…4806.040.76623-111.84755153"UT""America/Denver"4727.71997-01-01 00:00:00 UTC2024-12-22 04:05:00 UTCfalsefalsefalsetrue
"WBB"2024-12-22 03:33:00 UTC"air_temp"1false2.028null"Celsius"1"U of U William Browning Buildi…4806.040.76623-111.84755153"UT""America/Denver"4727.71997-01-01 00:00:00 UTC2024-12-22 04:05:00 UTCfalsefalsefalsetrue
"WBB"2024-12-22 03:34:00 UTC"air_temp"1false2.139null"Celsius"1"U of U William Browning Buildi…4806.040.76623-111.84755153"UT""America/Denver"4727.71997-01-01 00:00:00 UTC2024-12-22 04:05:00 UTCfalsefalsefalsetrue
"WBB"2024-12-22 03:35:00 UTC"air_temp"1false2.222null"Celsius"1"U of U William Browning Buildi…4806.040.76623-111.84755153"UT""America/Denver"4727.71997-01-01 00:00:00 UTC2024-12-22 04:05:00 UTCfalsefalsefalsetrue
……………………………………………………………
"WBB"2024-12-22 04:21:00 UTC"wind_cardinal_direction"1truenull"N""wind_cardinal_direction"1"U of U William Browning Buildi…4806.040.76623-111.84755153"UT""America/Denver"4727.71997-01-01 00:00:00 UTC2024-12-22 04:05:00 UTCfalsefalsefalsetrue
"WBB"2024-12-22 04:22:00 UTC"wind_cardinal_direction"1truenull"NNE""wind_cardinal_direction"1"U of U William Browning Buildi…4806.040.76623-111.84755153"UT""America/Denver"4727.71997-01-01 00:00:00 UTC2024-12-22 04:05:00 UTCfalsefalsefalsetrue
"WBB"2024-12-22 04:23:00 UTC"wind_cardinal_direction"1truenull"NE""wind_cardinal_direction"1"U of U William Browning Buildi…4806.040.76623-111.84755153"UT""America/Denver"4727.71997-01-01 00:00:00 UTC2024-12-22 04:05:00 UTCfalsefalsefalsetrue
"WBB"2024-12-22 04:24:00 UTC"wind_cardinal_direction"1truenull"NE""wind_cardinal_direction"1"U of U William Browning Buildi…4806.040.76623-111.84755153"UT""America/Denver"4727.71997-01-01 00:00:00 UTC2024-12-22 04:05:00 UTCfalsefalsefalsetrue
"WBB"2024-12-22 04:25:00 UTC"wind_cardinal_direction"1truenull"NE""wind_cardinal_direction"1"U of U William Browning Buildi…4806.040.76623-111.84755153"UT""America/Denver"4727.71997-01-01 00:00:00 UTC2024-12-22 04:05:00 UTCfalsefalsefalsetrue
" + ], + "text/plain": [ + "shape: (1_155, 23)\n", + "┌──────┬────────────┬────────────┬────────────┬───┬────────────┬───────────┬───────────┬───────────┐\n", + "│ stid ┆ date_time ┆ variable ┆ sensor_ind ┆ … ┆ qc_flagged ┆ is_restri ┆ restricte ┆ is_active │\n", + "│ --- ┆ --- ┆ --- ┆ ex ┆ ┆ --- ┆ cted ┆ d_metadat ┆ --- │\n", + "│ str ┆ datetime[μ ┆ str ┆ --- ┆ ┆ bool ┆ --- ┆ a ┆ bool │\n", + "│ ┆ s, UTC] ┆ ┆ u32 ┆ ┆ ┆ bool ┆ --- ┆ │\n", + "│ ┆ ┆ ┆ ┆ ┆ ┆ ┆ bool ┆ │\n", + "╞══════╪════════════╪════════════╪════════════╪═══╪════════════╪═══════════╪═══════════╪═══════════╡\n", + "│ WBB ┆ 2024-12-22 ┆ air_temp ┆ 1 ┆ … ┆ false ┆ false ┆ false ┆ true │\n", + "│ ┆ 03:31:00 ┆ ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ ┆ UTC ┆ ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ WBB ┆ 2024-12-22 ┆ air_temp ┆ 1 ┆ … ┆ false ┆ false ┆ false ┆ true │\n", + "│ ┆ 03:32:00 ┆ ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ ┆ UTC ┆ ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ WBB ┆ 2024-12-22 ┆ air_temp ┆ 1 ┆ … ┆ false ┆ false ┆ false ┆ true │\n", + "│ ┆ 03:33:00 ┆ ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ ┆ UTC ┆ ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ WBB ┆ 2024-12-22 ┆ air_temp ┆ 1 ┆ … ┆ false ┆ false ┆ false ┆ true │\n", + "│ ┆ 03:34:00 ┆ ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ ┆ UTC ┆ ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ WBB ┆ 2024-12-22 ┆ air_temp ┆ 1 ┆ … ┆ false ┆ false ┆ false ┆ true │\n", + "│ ┆ 03:35:00 ┆ ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ ┆ UTC ┆ ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", + "│ WBB ┆ 2024-12-22 ┆ wind_cardi ┆ 1 ┆ … ┆ false ┆ false ┆ false ┆ true │\n", + "│ ┆ 04:21:00 ┆ nal_direct ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ ┆ UTC ┆ ion ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ WBB ┆ 2024-12-22 ┆ wind_cardi ┆ 1 ┆ … ┆ false ┆ false ┆ false ┆ true │\n", + "│ ┆ 04:22:00 ┆ nal_direct ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ ┆ UTC ┆ ion ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ WBB ┆ 2024-12-22 ┆ wind_cardi ┆ 1 ┆ … ┆ false ┆ false ┆ false ┆ true │\n", + "│ ┆ 04:23:00 ┆ nal_direct ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ ┆ UTC ┆ ion ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ WBB ┆ 2024-12-22 ┆ wind_cardi ┆ 1 ┆ … ┆ false ┆ false ┆ false ┆ true │\n", + "│ ┆ 04:24:00 ┆ nal_direct ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ ┆ UTC ┆ ion ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ WBB ┆ 2024-12-22 ┆ wind_cardi ┆ 1 ┆ … ┆ false ┆ false ┆ false ┆ true │\n", + "│ ┆ 04:25:00 ┆ nal_direct ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ ┆ UTC ┆ ion ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "└──────┴────────────┴────────────┴────────────┴───┴────────────┴───────────┴───────────┴───────────┘" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Get DataFrom Synoptic\n", + "df = synoptic.TimeSeries(stid=\"ubkbk,wbb\", recent=60).df()\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "shape: (790, 11)
Message_TypeStation_IDValid_TimeLatLonElevationVariable_NameLevelHeightQC_StringObservation_Value
strstrstrf64f64f64strnullnullstrf64
"MESONET""WBB""20241222_033100"40.76623-111.847551464.8688"TMP"nullnull"passed"1.978
"MESONET""WBB""20241222_033200"40.76623-111.847551464.8688"TMP"nullnull"passed"1.972
"MESONET""WBB""20241222_033300"40.76623-111.847551464.8688"TMP"nullnull"passed"2.028
"MESONET""WBB""20241222_033400"40.76623-111.847551464.8688"TMP"nullnull"passed"2.139
"MESONET""WBB""20241222_033500"40.76623-111.847551464.8688"TMP"nullnull"passed"2.222
……………………………
"MESONET""WBB""20241222_042100"40.76623-111.847551464.8688"DPT"nullnull"passed"-1.48
"MESONET""WBB""20241222_042200"40.76623-111.847551464.8688"DPT"nullnull"passed"-1.53
"MESONET""WBB""20241222_042300"40.76623-111.847551464.8688"DPT"nullnull"passed"-1.54
"MESONET""WBB""20241222_042400"40.76623-111.847551464.8688"DPT"nullnull"passed"-1.47
"MESONET""WBB""20241222_042500"40.76623-111.847551464.8688"DPT"nullnull"passed"-1.38
" + ], + "text/plain": [ + "shape: (790, 11)\n", + "┌─────────────┬────────────┬─────────────┬──────────┬───┬───────┬────────┬───────────┬─────────────┐\n", + "│ Message_Typ ┆ Station_ID ┆ Valid_Time ┆ Lat ┆ … ┆ Level ┆ Height ┆ QC_String ┆ Observation │\n", + "│ e ┆ --- ┆ --- ┆ --- ┆ ┆ --- ┆ --- ┆ --- ┆ _Value │\n", + "│ --- ┆ str ┆ str ┆ f64 ┆ ┆ null ┆ null ┆ str ┆ --- │\n", + "│ str ┆ ┆ ┆ ┆ ┆ ┆ ┆ ┆ f64 │\n", + "╞═════════════╪════════════╪═════════════╪══════════╪═══╪═══════╪════════╪═══════════╪═════════════╡\n", + "│ MESONET ┆ WBB ┆ 20241222_03 ┆ 40.76623 ┆ … ┆ null ┆ null ┆ passed ┆ 1.978 │\n", + "│ ┆ ┆ 3100 ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ MESONET ┆ WBB ┆ 20241222_03 ┆ 40.76623 ┆ … ┆ null ┆ null ┆ passed ┆ 1.972 │\n", + "│ ┆ ┆ 3200 ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ MESONET ┆ WBB ┆ 20241222_03 ┆ 40.76623 ┆ … ┆ null ┆ null ┆ passed ┆ 2.028 │\n", + "│ ┆ ┆ 3300 ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ MESONET ┆ WBB ┆ 20241222_03 ┆ 40.76623 ┆ … ┆ null ┆ null ┆ passed ┆ 2.139 │\n", + "│ ┆ ┆ 3400 ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ MESONET ┆ WBB ┆ 20241222_03 ┆ 40.76623 ┆ … ┆ null ┆ null ┆ passed ┆ 2.222 │\n", + "│ ┆ ┆ 3500 ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n", + "│ MESONET ┆ WBB ┆ 20241222_04 ┆ 40.76623 ┆ … ┆ null ┆ null ┆ passed ┆ -1.48 │\n", + "│ ┆ ┆ 2100 ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ MESONET ┆ WBB ┆ 20241222_04 ┆ 40.76623 ┆ … ┆ null ┆ null ┆ passed ┆ -1.53 │\n", + "│ ┆ ┆ 2200 ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ MESONET ┆ WBB ┆ 20241222_04 ┆ 40.76623 ┆ … ┆ null ┆ null ┆ passed ┆ -1.54 │\n", + "│ ┆ ┆ 2300 ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ MESONET ┆ WBB ┆ 20241222_04 ┆ 40.76623 ┆ … ┆ null ┆ null ┆ passed ┆ -1.47 │\n", + "│ ┆ ┆ 2400 ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "│ MESONET ┆ WBB ┆ 20241222_04 ┆ 40.76623 ┆ … ┆ null ┆ null ┆ passed ┆ -1.38 │\n", + "│ ┆ ┆ 2500 ┆ ┆ ┆ ┆ ┆ ┆ │\n", + "└─────────────┴────────────┴─────────────┴──────────┴───┴───────┴────────┴───────────┴─────────────┘" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Let's not consider any rows where the observed value is None\n", + "met = df.filter(~pl.col(\"value\").is_null())\n", + "\n", + "# MET expects the data to be in 11 columns. This just requires some renaming.\n", + "# Reference: https://met.readthedocs.io/en/latest/Users_Guide/reformat_point.html#ascii2nc-tool\n", + "met = met.select(\n", + " pl.lit(\"MESONET\").alias(\"Message_Type\"),\n", + " pl.col(\"stid\").alias(\"Station_ID\"),\n", + " pl.col(\"date_time\").dt.strftime(\"%Y%m%d_%H%M%S\").alias(\"Valid_Time\"),\n", + " pl.col(\"latitude\").alias(\"Lat\"),\n", + " pl.col(\"longitude\").alias(\"Lon\"),\n", + " pl.col(\"elevation\").alias(\"Elevation\") * 0.3048, # feet to meters\n", + " pl.col(\"variable\").alias(\"Variable_Name\"),\n", + " pl.lit(None).alias(\"Level\"),\n", + " pl.lit(None).alias(\"Height\"),\n", + " pl.when(pl.col(\"qc_flagged\"))\n", + " .then(pl.lit(\"flagged\"))\n", + " .otherwise(pl.lit(\"passed\"))\n", + " .alias(\"QC_String\"),\n", + " pl.col(\"value\").alias(\"Observation_Value\"),\n", + ")\n", + "\n", + "# Now let's replace Synoptic's variable name with the GRIB short name\n", + "# TODO: List is incomplete\n", + "met = met.with_columns(\n", + " pl.col(\"Variable_Name\").replace(\n", + " {\n", + " \"air_temp\": \"TMP\",\n", + " \"relative_humidity\": \"RH\",\n", + " \"dew_point_temperature\": \"DPT\",\n", + " \"wind_speed\": \"WIND\",\n", + " \"wind_direction\": \"WDIR\",\n", + " \"sea_level_pressure\": \"PRMSL\",\n", + " \"pressure\": \"PRES\",\n", + " }\n", + " )\n", + ")\n", + "\n", + "# Let's see what we have now\n", + "met\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [], + "source": [ + "# And finally, rrite this to an ASCII file\n", + "# TODO: The file written is space-delimitated, not fixed with.\n", + "# TODO: Is that OK for MET? If not, need to use formatted np.savetxt.\n", + "met.with_columns(pl.all().cast(str)).fill_null(\"NA\").write_csv(\n", + " \"sample_ascii_for_met2.txt\",\n", + " separator=\" \",\n", + " include_header=False,\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "synoptic2", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}