From 5ffcc3a28b972296af46993442c62bb6b5eb431d Mon Sep 17 00:00:00 2001 From: tomvothecoder Date: Fri, 5 Jan 2024 10:44:17 -0800 Subject: [PATCH] Update notebook and script --- .../660-cosp-histogram/660_cosp_histogram.py | 7 +- .../660_cosp_histogram_test.ipynb | 1642 +++++++++++++++++ 2 files changed, 1647 insertions(+), 2 deletions(-) create mode 100644 auxiliary_tools/cdat_regression_testing/660-cosp-histogram/660_cosp_histogram_test.ipynb diff --git a/auxiliary_tools/cdat_regression_testing/660-cosp-histogram/660_cosp_histogram.py b/auxiliary_tools/cdat_regression_testing/660-cosp-histogram/660_cosp_histogram.py index c20198ee05..c4d93ef7b1 100644 --- a/auxiliary_tools/cdat_regression_testing/660-cosp-histogram/660_cosp_histogram.py +++ b/auxiliary_tools/cdat_regression_testing/660-cosp-histogram/660_cosp_histogram.py @@ -28,10 +28,13 @@ param.results_dir = os.path.join(prefix, "660-cosp-histogram", param.sets[0]) param.multiprocessing = False +# Make sure to save the NetCDF for metrics comparison. +param.save_netcdf = True + # %% CFG_PATH = ( - "/global/u2/v/vo13/E3SM-Project/e3sm_diags/auxiliary_tools/" - "cdat_regression_testing/660-cosp-histogram/model_vs_model/660-cosp-histogram.cfg" + "/global/u2/v/vo13/E3SM-Project/e3sm_diags/auxiliary_tools/cdat_regression_testing/" + "660-cosp-histogram/660-cosp-histogram.cfg" ) sys.argv.extend(["-d", CFG_PATH]) runner.run_diags([param]) diff --git a/auxiliary_tools/cdat_regression_testing/660-cosp-histogram/660_cosp_histogram_test.ipynb b/auxiliary_tools/cdat_regression_testing/660-cosp-histogram/660_cosp_histogram_test.ipynb new file mode 100644 index 0000000000..6da8ab4dae --- /dev/null +++ b/auxiliary_tools/cdat_regression_testing/660-cosp-histogram/660_cosp_histogram_test.ipynb @@ -0,0 +1,1642 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# CDAT Migration Regression Testing Notebook\n", + "\n", + "This notebook is used to perform regression testing between the development and\n", + "production versions of a diagnostic set.\n", + "\n", + "## How it works\n", + "\n", + "It compares the relative differences (%) between two sets of `.json` files in two\n", + "separate directories, one for the refactored code and the other for the `main` branch.\n", + "\n", + "It will display metrics values with relative differences >= 2%. Relative differences are used instead of absolute differences because:\n", + "\n", + "- Relative differences are in percentages, which shows the scale of the differences.\n", + "- Absolute differences are just a raw number that doesn't factor in\n", + " floating point size (e.g., 100.00 vs. 0.0001), which can be misleading.\n", + "\n", + "## How to use\n", + "\n", + "PREREQUISITE: The diagnostic set's metrics stored in `.json` files in two directories\n", + "(dev and `main` branches).\n", + "\n", + "1. Make a copy of this notebook under `auxiliary_tools/cdat_regression_testing/`.\n", + "2. Run `mamba create -n cdat_regression_test -y -c conda-forge \"python<3.12\" xarray dask pandas matplotlib-base ipykernel`\n", + "3. Run `mamba activate cdat_regression_test`\n", + "4. Update `DEV_PATH` and `MAIN_PATH` in the copy of your notebook.\n", + "5. Run all cells IN ORDER.\n", + "6. Review results for any outstanding differences (>= 2%).\n", + " - Debug these differences (e.g., bug in metrics functions, incorrect variable references, etc.)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup Code\n" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "metadata": {}, + "outputs": [], + "source": [ + "import glob\n", + "from auxiliary_tools.cdat_regression_testing.utils import (\n", + " get_metrics,\n", + " get_rel_diffs,\n", + " get_num_metrics_above_diff_thres,\n", + " highlight_large_diffs,\n", + " sort_columns,\n", + " update_diffs_to_pct,\n", + " PERCENTAGE_COLUMNS,\n", + ")\n", + "import xarray as xr\n", + "\n", + "# TODO: Update DEV_RESULTS and MAIN_RESULTS to your diagnostic sets.\n", + "DEV_PATH = \"/global/cfs/cdirs/e3sm/www/vo13/cdat-migration-test/660-cosp-histogram/cosp_histogram/cosp_histogram/model_vs_model\"\n", + "MAIN_PATH = \"/global/cfs/cdirs/e3sm/www/vo13/cdat-migration-test/main-cosp-histogram/cosp_histogram/cosp_histogram/model_vs_model\"\n", + "\n", + "DEV_GLOB = sorted(glob.glob(DEV_PATH + \"/*.nc\"))\n", + "MAIN_GLOB = sorted(glob.glob(MAIN_PATH + \"/*.nc\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[array(['/global/cfs/cdirs/e3sm/www/vo13/cdat-migration-test/main-cosp-histogram/cosp_histogram/cosp_histogram/model_vs_model/MISRCOSP-COSP_HISTOGRAM_MISR-ANN-global_diff.nc',\n", + " '/global/cfs/cdirs/e3sm/www/vo13/cdat-migration-test/main-cosp-histogram/cosp_histogram/cosp_histogram/model_vs_model/MISRCOSP-COSP_HISTOGRAM_MISR-ANN-global_ref.nc',\n", + " '/global/cfs/cdirs/e3sm/www/vo13/cdat-migration-test/main-cosp-histogram/cosp_histogram/cosp_histogram/model_vs_model/MISRCOSP-COSP_HISTOGRAM_MISR-ANN-global_test.nc'],\n", + " dtype='\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset>\n",
+       "Dimensions:                   (axis_4624: 15, axis_4623: 6, misr_cth: 15,\n",
+       "                               misr_tau: 6, bound: 2, cosp_htmisr: 15,\n",
+       "                               cosp_tau: 6)\n",
+       "Coordinates:\n",
+       "  * axis_4624                 (axis_4624) int32 0 1 2 3 4 5 ... 9 10 11 12 13 14\n",
+       "  * axis_4623                 (axis_4623) int32 0 1 2 3 4 5\n",
+       "  * misr_cth                  (misr_cth) float32 0.25 0.75 1.25 ... 16.0 20.0\n",
+       "  * misr_tau                  (misr_tau) float32 0.8 2.45 6.5 16.2 41.5 100.0\n",
+       "  * cosp_htmisr               (cosp_htmisr) float64 0.25 0.75 1.25 ... 16.0 58.0\n",
+       "  * cosp_tau                  (cosp_tau) float64 0.8 2.45 6.5 16.2 41.5 219.5\n",
+       "Dimensions without coordinates: bound\n",
+       "Data variables:\n",
+       "    COSP_HISTOGRAM_MISR_diff  (axis_4624, axis_4623) float64 dask.array<chunksize=(15, 6), meta=np.ndarray>\n",
+       "    misr_tau_bnds             (misr_tau, bound) float32 dask.array<chunksize=(6, 2), meta=np.ndarray>\n",
+       "    COSP_HISTOGRAM_MISR_ref   (misr_cth, misr_tau) float64 dask.array<chunksize=(15, 6), meta=np.ndarray>\n",
+       "    cosp_htmisr_bnds          (cosp_htmisr, bound) float64 dask.array<chunksize=(15, 2), meta=np.ndarray>\n",
+       "    cosp_tau_bnds             (cosp_tau, bound) float64 dask.array<chunksize=(6, 2), meta=np.ndarray>\n",
+       "    COSP_HISTOGRAM_MISR_test  (cosp_htmisr, cosp_tau) float64 dask.array<chunksize=(15, 6), meta=np.ndarray>\n",
+       "Attributes:\n",
+       "    Conventions:  CF-1.0
" + ], + "text/plain": [ + "\n", + "Dimensions: (axis_4624: 15, axis_4623: 6, misr_cth: 15,\n", + " misr_tau: 6, bound: 2, cosp_htmisr: 15,\n", + " cosp_tau: 6)\n", + "Coordinates:\n", + " * axis_4624 (axis_4624) int32 0 1 2 3 4 5 ... 9 10 11 12 13 14\n", + " * axis_4623 (axis_4623) int32 0 1 2 3 4 5\n", + " * misr_cth (misr_cth) float32 0.25 0.75 1.25 ... 16.0 20.0\n", + " * misr_tau (misr_tau) float32 0.8 2.45 6.5 16.2 41.5 100.0\n", + " * cosp_htmisr (cosp_htmisr) float64 0.25 0.75 1.25 ... 16.0 58.0\n", + " * cosp_tau (cosp_tau) float64 0.8 2.45 6.5 16.2 41.5 219.5\n", + "Dimensions without coordinates: bound\n", + "Data variables:\n", + " COSP_HISTOGRAM_MISR_diff (axis_4624, axis_4623) float64 dask.array\n", + " misr_tau_bnds (misr_tau, bound) float32 dask.array\n", + " COSP_HISTOGRAM_MISR_ref (misr_cth, misr_tau) float64 dask.array\n", + " cosp_htmisr_bnds (cosp_htmisr, bound) float64 dask.array\n", + " cosp_tau_bnds (cosp_tau, bound) float64 dask.array\n", + " COSP_HISTOGRAM_MISR_test (cosp_htmisr, cosp_tau) float64 dask.array\n", + "Attributes:\n", + " Conventions: CF-1.0" + ] + }, + "execution_count": 75, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "xr.open_mfdataset(MAIN_GLOB_SPLIT[0])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Get the metrics for the development and `main` branches and their differences.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "array([[ nan, nan, 0.730555, nan, 0.075411, nan],\n", + " [ nan, nan, 1.551659, nan, 0.201809, nan],\n", + " [ nan, nan, 2.781434, nan, 0.434809, nan],\n", + " [ nan, nan, 2.915085, nan, 0.531622, nan],\n", + " [ nan, nan, 1.876356, nan, 0.423032, nan],\n", + " [ nan, nan, 0.998693, nan, 0.294345, nan],\n", + " [ nan, nan, 1.097111, nan, 0.421908, nan],\n", + " [ nan, nan, 0.768678, nan, 0.357684, nan],\n", + " [ nan, nan, 1.510228, nan, 0.715053, nan],\n", + " [ nan, nan, 0.929532, nan, 0.543281, nan],\n", + " [ nan, nan, 0.744868, nan, 0.469284, nan],\n", + " [ nan, nan, 0.653314, nan, 0.447334, nan],\n", + " [ nan, nan, 0.290538, nan, 0.2414 , nan],\n", + " [ nan, nan, 0.065314, nan, 0.050509, nan],\n", + " [ nan, nan, nan, nan, nan, nan]])\n", + "Coordinates:\n", + " * misr_cth (misr_cth) float64 0.25 0.75 1.25 1.75 ... 12.0 14.0 16.0 58.0\n", + " * misr_tau (misr_tau) float64 0.8 2.45 6.5 16.2 41.5 219.5\n", + "Attributes:\n", + " long_name: cloud occurrence histogram\n", + " units: percent\n", + "\n", + "array([[2.19074876, 0.87516499, 0.73055494, 0.43687444, 0.07541108,\n", + " 0.01072699],\n", + " [2.62562678, 1.56597614, 1.55165947, 1.07405153, 0.20180927,\n", + " 0.01871206],\n", + " [3.18852993, 2.56138817, 2.78143362, 2.00331225, 0.43480856,\n", + " 0.04105982],\n", + " [2.042739 , 2.36524911, 2.9150847 , 2.11246131, 0.5316223 ,\n", + " 0.065312 ],\n", + " [1.09754604, 1.43951883, 1.87635564, 1.45739415, 0.42303243,\n", + " 0.07803713],\n", + " [0.50887333, 0.70175208, 0.99869349, 0.90419338, 0.2943452 ,\n", + " 0.07056132],\n", + " [0.47720912, 0.6857048 , 1.0971112 , 1.16630678, 0.42190844,\n", + " 0.12197551],\n", + " [0.30777588, 0.4676754 , 0.76867767, 0.87802087, 0.3576838 ,\n", + " 0.1156163 ],\n", + " [0.64890214, 1.01728449, 1.51022843, 1.62806136, 0.71505283,\n", + " 0.2429296 ],\n", + " [0.52744384, 0.70421976, 0.92953173, 1.03025704, 0.54328104,\n", + " 0.19842325],\n", + " [0.59881938, 0.68390832, 0.74486766, 0.76371307, 0.46928398,\n", + " 0.17861999],\n", + " [0.67346366, 0.74796183, 0.65331393, 0.58644159, 0.44733404,\n", + " 0.24139218],\n", + " [0.38962577, 0.4088059 , 0.29053768, 0.23226709, 0.24139989,\n", + " 0.27580595],\n", + " [0.1002934 , 0.09226598, 0.06531426, 0.04846917, 0.05050884,\n", + " 0.10334305],\n", + " [0.06087834, 0.04121329, 0.03119792, 0.01935536, 0.00861623,\n", + " 0.00999498]])\n", + "Coordinates:\n", + " * misr_cth (misr_cth) float32 0.25 0.75 1.25 1.75 ... 12.0 14.0 16.0 20.0\n", + " * misr_tau (misr_tau) float32 0.8 2.45 6.5 16.2 41.5 100.0\n", + "Attributes:\n", + " long_name: cloud occurrence histogram\n", + " units: percent\n", + "\n", + "array([[ nan, nan, 0.730555, nan, 0.075411, nan],\n", + " [ nan, nan, 1.551659, nan, 0.201809, nan],\n", + " [ nan, nan, 2.781434, nan, 0.434809, nan],\n", + " [ nan, nan, 2.915085, nan, 0.531622, nan],\n", + " [ nan, nan, 1.876356, nan, 0.423032, nan],\n", + " [ nan, nan, 0.998693, nan, 0.294345, nan],\n", + " [ nan, nan, 1.097111, nan, 0.421908, nan],\n", + " [ nan, nan, 0.768678, nan, 0.357684, nan],\n", + " [ nan, nan, 1.510228, nan, 0.715053, nan],\n", + " [ nan, nan, 0.929532, nan, 0.543281, nan],\n", + " [ nan, nan, 0.744868, nan, 0.469284, nan],\n", + " [ nan, nan, 0.653314, nan, 0.447334, nan],\n", + " [ nan, nan, 0.290538, nan, 0.2414 , nan],\n", + " [ nan, nan, 0.065314, nan, 0.050509, nan],\n", + " [ nan, nan, nan, nan, nan, nan]])\n", + "Coordinates:\n", + " * misr_cth (misr_cth) float64 0.25 0.75 1.25 1.75 ... 12.0 14.0 16.0 58.0\n", + " * misr_tau (misr_tau) float64 0.8 2.45 6.5 16.2 41.5 219.5\n", + "Attributes:\n", + " long_name: cloud occurrence histogram\n", + " units: percent\n", + "\n", + "array([[2.19074876, 0.87516499, 0.73055494, 0.43687444, 0.07541108,\n", + " 0.01072699],\n", + " [2.62562678, 1.56597614, 1.55165947, 1.07405153, 0.20180927,\n", + " 0.01871206],\n", + " [3.18852993, 2.56138817, 2.78143362, 2.00331225, 0.43480856,\n", + " 0.04105982],\n", + " [2.042739 , 2.36524911, 2.9150847 , 2.11246131, 0.5316223 ,\n", + " 0.065312 ],\n", + " [1.09754604, 1.43951883, 1.87635564, 1.45739415, 0.42303243,\n", + " 0.07803713],\n", + " [0.50887333, 0.70175208, 0.99869349, 0.90419338, 0.2943452 ,\n", + " 0.07056132],\n", + " [0.47720912, 0.6857048 , 1.0971112 , 1.16630678, 0.42190844,\n", + " 0.12197551],\n", + " [0.30777588, 0.4676754 , 0.76867767, 0.87802087, 0.3576838 ,\n", + " 0.1156163 ],\n", + " [0.64890214, 1.01728449, 1.51022843, 1.62806136, 0.71505283,\n", + " 0.2429296 ],\n", + " [0.52744384, 0.70421976, 0.92953173, 1.03025704, 0.54328104,\n", + " 0.19842325],\n", + " [0.59881938, 0.68390832, 0.74486766, 0.76371307, 0.46928398,\n", + " 0.17861999],\n", + " [0.67346366, 0.74796183, 0.65331393, 0.58644159, 0.44733404,\n", + " 0.24139218],\n", + " [0.38962577, 0.4088059 , 0.29053768, 0.23226709, 0.24139989,\n", + " 0.27580595],\n", + " [0.1002934 , 0.09226598, 0.06531426, 0.04846917, 0.05050884,\n", + " 0.10334305],\n", + " [0.06087834, 0.04121329, 0.03119792, 0.01935536, 0.00861623,\n", + " 0.00999498]])\n", + "Coordinates:\n", + " * misr_cth (misr_cth) float32 0.25 0.75 1.25 1.75 ... 12.0 14.0 16.0 20.0\n", + " * misr_tau (misr_tau) float32 0.8 2.45 6.5 16.2 41.5 100.0\n", + "Attributes:\n", + " long_name: cloud occurrence histogram\n", + " units: percent\n", + "\n", + "array([[ nan, nan, 0.730555, nan, 0.075411, nan],\n", + " [ nan, nan, 1.551659, nan, 0.201809, nan],\n", + " [ nan, nan, 2.781434, nan, 0.434809, nan],\n", + " [ nan, nan, 2.915085, nan, 0.531622, nan],\n", + " [ nan, nan, 1.876356, nan, 0.423032, nan],\n", + " [ nan, nan, 0.998693, nan, 0.294345, nan],\n", + " [ nan, nan, 1.097111, nan, 0.421908, nan],\n", + " [ nan, nan, 0.768678, nan, 0.357684, nan],\n", + " [ nan, nan, 1.510228, nan, 0.715053, nan],\n", + " [ nan, nan, 0.929532, nan, 0.543281, nan],\n", + " [ nan, nan, 0.744868, nan, 0.469284, nan],\n", + " [ nan, nan, 0.653314, nan, 0.447334, nan],\n", + " [ nan, nan, 0.290538, nan, 0.2414 , nan],\n", + " [ nan, nan, 0.065314, nan, 0.050509, nan],\n", + " [ nan, nan, nan, nan, nan, nan]])\n", + "Coordinates:\n", + " * misr_cth (misr_cth) float64 0.25 0.75 1.25 1.75 ... 12.0 14.0 16.0 58.0\n", + " * misr_tau (misr_tau) float64 0.8 2.45 6.5 16.2 41.5 219.5\n", + "Attributes:\n", + " long_name: cloud occurrence histogram\n", + " units: percent\n", + "\n", + "array([[2.19074876, 0.87516499, 0.73055494, 0.43687444, 0.07541108,\n", + " 0.01072699],\n", + " [2.62562678, 1.56597614, 1.55165947, 1.07405153, 0.20180927,\n", + " 0.01871206],\n", + " [3.18852993, 2.56138817, 2.78143362, 2.00331225, 0.43480856,\n", + " 0.04105982],\n", + " [2.042739 , 2.36524911, 2.9150847 , 2.11246131, 0.5316223 ,\n", + " 0.065312 ],\n", + " [1.09754604, 1.43951883, 1.87635564, 1.45739415, 0.42303243,\n", + " 0.07803713],\n", + " [0.50887333, 0.70175208, 0.99869349, 0.90419338, 0.2943452 ,\n", + " 0.07056132],\n", + " [0.47720912, 0.6857048 , 1.0971112 , 1.16630678, 0.42190844,\n", + " 0.12197551],\n", + " [0.30777588, 0.4676754 , 0.76867767, 0.87802087, 0.3576838 ,\n", + " 0.1156163 ],\n", + " [0.64890214, 1.01728449, 1.51022843, 1.62806136, 0.71505283,\n", + " 0.2429296 ],\n", + " [0.52744384, 0.70421976, 0.92953173, 1.03025704, 0.54328104,\n", + " 0.19842325],\n", + " [0.59881938, 0.68390832, 0.74486766, 0.76371307, 0.46928398,\n", + " 0.17861999],\n", + " [0.67346366, 0.74796183, 0.65331393, 0.58644159, 0.44733404,\n", + " 0.24139218],\n", + " [0.38962577, 0.4088059 , 0.29053768, 0.23226709, 0.24139989,\n", + " 0.27580595],\n", + " [0.1002934 , 0.09226598, 0.06531426, 0.04846917, 0.05050884,\n", + " 0.10334305],\n", + " [0.06087834, 0.04121329, 0.03119792, 0.01935536, 0.00861623,\n", + " 0.00999498]])\n", + "Coordinates:\n", + " * misr_cth (misr_cth) float32 0.25 0.75 1.25 1.75 ... 12.0 14.0 16.0 20.0\n", + " * misr_tau (misr_tau) float32 0.8 2.45 6.5 16.2 41.5 100.0\n", + "Attributes:\n", + " long_name: cloud occurrence histogram\n", + " units: percent\n", + "\n", + "array([[ nan, nan, 0.730555, nan, 0.075411, nan],\n", + " [ nan, nan, 1.551659, nan, 0.201809, nan],\n", + " [ nan, nan, 2.781434, nan, 0.434809, nan],\n", + " [ nan, nan, 2.915085, nan, 0.531622, nan],\n", + " [ nan, nan, 1.876356, nan, 0.423032, nan],\n", + " [ nan, nan, 0.998693, nan, 0.294345, nan],\n", + " [ nan, nan, 1.097111, nan, 0.421908, nan],\n", + " [ nan, nan, 0.768678, nan, 0.357684, nan],\n", + " [ nan, nan, 1.510228, nan, 0.715053, nan],\n", + " [ nan, nan, 0.929532, nan, 0.543281, nan],\n", + " [ nan, nan, 0.744868, nan, 0.469284, nan],\n", + " [ nan, nan, 0.653314, nan, 0.447334, nan],\n", + " [ nan, nan, 0.290538, nan, 0.2414 , nan],\n", + " [ nan, nan, 0.065314, nan, 0.050509, nan],\n", + " [ nan, nan, nan, nan, nan, nan]])\n", + "Coordinates:\n", + " * misr_cth (misr_cth) float64 0.25 0.75 1.25 1.75 ... 12.0 14.0 16.0 58.0\n", + " * misr_tau (misr_tau) float64 0.8 2.45 6.5 16.2 41.5 219.5\n", + "Attributes:\n", + " long_name: cloud occurrence histogram\n", + " units: percent\n", + "\n", + "array([[2.19074876, 0.87516499, 0.73055494, 0.43687444, 0.07541108,\n", + " 0.01072699],\n", + " [2.62562678, 1.56597614, 1.55165947, 1.07405153, 0.20180927,\n", + " 0.01871206],\n", + " [3.18852993, 2.56138817, 2.78143362, 2.00331225, 0.43480856,\n", + " 0.04105982],\n", + " [2.042739 , 2.36524911, 2.9150847 , 2.11246131, 0.5316223 ,\n", + " 0.065312 ],\n", + " [1.09754604, 1.43951883, 1.87635564, 1.45739415, 0.42303243,\n", + " 0.07803713],\n", + " [0.50887333, 0.70175208, 0.99869349, 0.90419338, 0.2943452 ,\n", + " 0.07056132],\n", + " [0.47720912, 0.6857048 , 1.0971112 , 1.16630678, 0.42190844,\n", + " 0.12197551],\n", + " [0.30777588, 0.4676754 , 0.76867767, 0.87802087, 0.3576838 ,\n", + " 0.1156163 ],\n", + " [0.64890214, 1.01728449, 1.51022843, 1.62806136, 0.71505283,\n", + " 0.2429296 ],\n", + " [0.52744384, 0.70421976, 0.92953173, 1.03025704, 0.54328104,\n", + " 0.19842325],\n", + " [0.59881938, 0.68390832, 0.74486766, 0.76371307, 0.46928398,\n", + " 0.17861999],\n", + " [0.67346366, 0.74796183, 0.65331393, 0.58644159, 0.44733404,\n", + " 0.24139218],\n", + " [0.38962577, 0.4088059 , 0.29053768, 0.23226709, 0.24139989,\n", + " 0.27580595],\n", + " [0.1002934 , 0.09226598, 0.06531426, 0.04846917, 0.05050884,\n", + " 0.10334305],\n", + " [0.06087834, 0.04121329, 0.03119792, 0.01935536, 0.00861623,\n", + " 0.00999498]])\n", + "Coordinates:\n", + " * misr_cth (misr_cth) float32 0.25 0.75 1.25 1.75 ... 12.0 14.0 16.0 20.0\n", + " * misr_tau (misr_tau) float32 0.8 2.45 6.5 16.2 41.5 100.0\n", + "Attributes:\n", + " long_name: cloud occurrence histogram\n", + " units: percent\n", + "\n", + "array([[ nan, nan, 0.730555, nan, 0.075411, nan],\n", + " [ nan, nan, 1.551659, nan, 0.201809, nan],\n", + " [ nan, nan, 2.781434, nan, 0.434809, nan],\n", + " [ nan, nan, 2.915085, nan, 0.531622, nan],\n", + " [ nan, nan, 1.876356, nan, 0.423032, nan],\n", + " [ nan, nan, 0.998693, nan, 0.294345, nan],\n", + " [ nan, nan, 1.097111, nan, 0.421908, nan],\n", + " [ nan, nan, 0.768678, nan, 0.357684, nan],\n", + " [ nan, nan, 1.510228, nan, 0.715053, nan],\n", + " [ nan, nan, 0.929532, nan, 0.543281, nan],\n", + " [ nan, nan, 0.744868, nan, 0.469284, nan],\n", + " [ nan, nan, 0.653314, nan, 0.447334, nan],\n", + " [ nan, nan, 0.290538, nan, 0.2414 , nan],\n", + " [ nan, nan, 0.065314, nan, 0.050509, nan],\n", + " [ nan, nan, nan, nan, nan, nan]])\n", + "Coordinates:\n", + " * misr_cth (misr_cth) float64 0.25 0.75 1.25 1.75 ... 12.0 14.0 16.0 58.0\n", + " * misr_tau (misr_tau) float64 0.8 2.45 6.5 16.2 41.5 219.5\n", + "Attributes:\n", + " long_name: cloud occurrence histogram\n", + " units: percent\n", + "\n", + "array([[2.19074876, 0.87516499, 0.73055494, 0.43687444, 0.07541108,\n", + " 0.01072699],\n", + " [2.62562678, 1.56597614, 1.55165947, 1.07405153, 0.20180927,\n", + " 0.01871206],\n", + " [3.18852993, 2.56138817, 2.78143362, 2.00331225, 0.43480856,\n", + " 0.04105982],\n", + " [2.042739 , 2.36524911, 2.9150847 , 2.11246131, 0.5316223 ,\n", + " 0.065312 ],\n", + " [1.09754604, 1.43951883, 1.87635564, 1.45739415, 0.42303243,\n", + " 0.07803713],\n", + " [0.50887333, 0.70175208, 0.99869349, 0.90419338, 0.2943452 ,\n", + " 0.07056132],\n", + " [0.47720912, 0.6857048 , 1.0971112 , 1.16630678, 0.42190844,\n", + " 0.12197551],\n", + " [0.30777588, 0.4676754 , 0.76867767, 0.87802087, 0.3576838 ,\n", + " 0.1156163 ],\n", + " [0.64890214, 1.01728449, 1.51022843, 1.62806136, 0.71505283,\n", + " 0.2429296 ],\n", + " [0.52744384, 0.70421976, 0.92953173, 1.03025704, 0.54328104,\n", + " 0.19842325],\n", + " [0.59881938, 0.68390832, 0.74486766, 0.76371307, 0.46928398,\n", + " 0.17861999],\n", + " [0.67346366, 0.74796183, 0.65331393, 0.58644159, 0.44733404,\n", + " 0.24139218],\n", + " [0.38962577, 0.4088059 , 0.29053768, 0.23226709, 0.24139989,\n", + " 0.27580595],\n", + " [0.1002934 , 0.09226598, 0.06531426, 0.04846917, 0.05050884,\n", + " 0.10334305],\n", + " [0.06087834, 0.04121329, 0.03119792, 0.01935536, 0.00861623,\n", + " 0.00999498]])\n", + "Coordinates:\n", + " * misr_cth (misr_cth) float32 0.25 0.75 1.25 1.75 ... 12.0 14.0 16.0 20.0\n", + " * misr_tau (misr_tau) float32 0.8 2.45 6.5 16.2 41.5 100.0\n", + "Attributes:\n", + " long_name: cloud occurrence histogram\n", + " units: percent\n" + ] + } + ], + "source": [ + "var_key = \"COSP_HISTOGRAM_MISR\"\n", + "\n", + "for file_a, file_b in zip(DEV_GLOB, MAIN_GLOB_SPLIT):\n", + " ds_a = xr.open_dataset(file_a)\n", + " ds_b = xr.open_mfdataset(file_b)\n", + "\n", + " np.testing.assert_allclose(\n", + " ds_a[f\"{var_key}_test\"].values, ds_b[f\"{var_key}_test\"].values\n", + " )\n", + " try:\n", + " np.testing.assert_allclose(\n", + " ds_a[f\"{var_key}_ref\"].values, ds_b[f\"{var_key}_ref\"].values\n", + " )\n", + " except AssertionError:\n", + " print(ds_a[f\"{var_key}_ref\"])\n", + " print(ds_b[f\"{var_key}_ref\"].load())\n", + " # np.testing.assert_allclose(\n", + " # ds_a[f\"{var_key}_diff\"].values, ds_b[f\"{var_key}_diff\"].values\n", + " # )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Filter differences to those above maximum threshold (2%).\n", + "\n", + "All values below maximum threshold will be labeled as `NaN`.\n", + "\n", + "- **If all cells in a row are NaN (< 2%)**, the entire row is dropped to make the results easier to parse.\n", + "- Any remaining NaN cells are below < 2% difference and **should be ignored**.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "df_metrics_diffs_thres = df_metrics_diffs[df_metrics_diffs >= 0.02]\n", + "df_metrics_diffs_thres = df_metrics_diffs_thres.dropna(\n", + " axis=0, how=\"all\", ignore_index=False\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Combine all DataFrames to get the final result.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [], + "source": [ + "df_metrics_all = pd.concat(\n", + " [df_metrics_dev.add_suffix(\"_dev\"), df_metrics_main.add_suffix(\"_main\")],\n", + " axis=1,\n", + " join=\"outer\",\n", + ")\n", + "df_final = df_metrics_diffs_thres.join(df_metrics_all)\n", + "df_final = sort_columns(df_final)\n", + "df_final = update_diffs_to_pct(df_final)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Review variables and metrics above difference threshold.\n", + "\n", + "- Red cells are differences >= 2%\n", + "- `nan` cells are differences < 2% and **should be ignored**\n" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "* Related variables ['FSNTOA', 'LHFLX', 'LWCF', 'NET_FLUX_SRF', 'PRECT', 'PSL', 'RESTOM', 'TREFHT']\n", + "* Number of metrics above 2% max threshold: 11 / 96\n" + ] + } + ], + "source": [ + "remove_metrics = [\"min\", \"max\"]\n", + "df_metrics_sub = df_final.reset_index(names=[\"var_key\", \"metric\"])\n", + "df_metrics_sub = df_metrics_sub[~df_metrics_sub.metric.isin(remove_metrics)]\n", + "get_num_metrics_above_diff_thres(df_metrics_all, df_metrics_sub)" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
 var_keymetrictest_devtest_maintest DIFF (%)ref_devref_mainref DIFF (%)test_regrid_devtest_regrid_maintest_regrid DIFF (%)ref_regrid_devref_regrid_mainref_regrid DIFF (%)misc_devmisc_mainmisc DIFF (%)
5FSNTOAmean239.859777240.001860nan241.439641241.544384nan239.859777240.001860nan241.439641241.544384nannannannan
8LHFLXmean88.37960988.470270nan88.96955088.976266nan88.37960988.470270nan88.96955088.976266nannannannan
11LWCFmean24.37322424.370539nan24.40669724.391579nan24.37322424.370539nan24.40669724.391579nannannannan
16NET_FLUX_SRFmean0.3940160.51633031.04%-0.0681860.068584200.58%0.3940160.51633031.04%-0.0681860.068584200.58%nannannan
19PRECTmean3.0538023.056760nan3.0748853.074978nan3.0538023.056760nan3.0748853.074978nannannannan
21PSLrmsenannannannannannannannannannannannan1.0428840.9799816.03%
23RESTOMmean0.4815490.65656036.34%0.0180410.162984803.40%0.4815490.65656036.34%0.0180410.162984803.40%nannannan
34TREFHTmean14.76994614.741707nan13.84201313.800258nan14.76994614.741707nan13.84201313.800258nannannannan
35TREFHTmean9.2142249.114572nan8.0833497.957917nan9.2142249.114572nan8.0833497.957917nannannannan
40TREFHTrmsenannannannannannannannannannannannan1.1607181.1799952.68%
41TREFHTrmsenannannannannannannannannannannannan1.3431691.3791412.68%
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "highlight_large_diffs(df_metrics_sub)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## `NET_FLUX_SRF` and `RESTOM` contain the highest differences and should be investigated further\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "cdat_regression_test", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}