diff --git a/.gitignore b/.gitignore index f1dd4a421..1409d072f 100644 --- a/.gitignore +++ b/.gitignore @@ -110,6 +110,7 @@ ENV/ # NetCDF files needed !e3sm_diags/driver/acme_ne30_ocean_land_mask.nc +!auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/*.nc # Folder for storing quality assurance files and notes qa/ \ No newline at end of file diff --git a/auxiliary_tools/cdat_regression_testing/759-slice-flag/671-diags.cfg b/auxiliary_tools/cdat_regression_testing/759-slice-flag/671-diags.cfg new file mode 100644 index 000000000..4c132d16d --- /dev/null +++ b/auxiliary_tools/cdat_regression_testing/759-slice-flag/671-diags.cfg @@ -0,0 +1,9 @@ +[#] +sets = ["lat_lon"] +case_id = "model_vs_model" +variables = ["TREFHT"] +regions = ["land"] +seasons = ["ANN", "01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12", "DJF", "MAM", "JJA", "SON"] +contour_levels = [-35, -30, -25, -20, -15, -10, -5, 0, 5, 10, 15, 20, 25, 30, 35, 40] +diff_levels = [-5, -4, -3, -2, -1, -0.5, -0.2, 0.2, 0.5, 1, 2, 3, 4, 5] +regrid_method = "bilinear" \ No newline at end of file diff --git a/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/ds_a.nc b/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/ds_a.nc new file mode 100644 index 000000000..e96267bf6 Binary files /dev/null and b/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/ds_a.nc differ diff --git a/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/ds_b.nc b/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/ds_b.nc new file mode 100644 index 000000000..15d131361 Binary files /dev/null and b/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/ds_b.nc differ diff --git a/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/ds_b_regrid.nc b/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/ds_b_regrid.nc new file mode 100644 index 000000000..203e3d285 Binary files /dev/null and b/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/ds_b_regrid.nc differ diff --git a/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/mv1.nc b/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/mv1.nc new file mode 100644 index 000000000..d3a0001c2 Binary files /dev/null and b/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/mv1.nc differ diff --git a/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/mv2.nc b/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/mv2.nc new file mode 100644 index 000000000..f2d3f441d Binary files /dev/null and b/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/mv2.nc differ diff --git a/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/mv2_regrid.nc b/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/mv2_regrid.nc new file mode 100644 index 000000000..f2d3f441d Binary files /dev/null and b/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/mv2_regrid.nc differ diff --git a/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/trefht.py b/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/trefht.py new file mode 100644 index 000000000..f96ee8c6d --- /dev/null +++ b/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/trefht.py @@ -0,0 +1,93 @@ +# %% +import os + +import cdms2 +import numpy as np +import xarray as xr +import xcdat as xc # noqa: F401 + +var_key = "TREFHT" +regrid_tool = "esmf" +regrid_method = "bilinear" + +dir_path = "/global/u2/v/vo13/E3SM-Project/e3sm_diags/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/" +fp_a = os.path.join(dir_path, "ds_a.nc") +fp_b = os.path.join(dir_path, "ds_b.nc") +fp_mv1 = os.path.join(dir_path, "mv1.nc") +fp_mv2 = os.path.join(dir_path, "mv2.nc") + + +# %% +# Regridding with xCDAT +ds_a = xr.open_dataset(fp_a) +ds_b = xr.open_dataset(fp_b) + +output_grid = ds_a.regridder.grid +ds_b_regrid = ds_b.regridder.horizontal( + var_key, output_grid, tool=f"x{regrid_tool}", method=regrid_method +) + +# Write out to netCDF for visualization +# ds_b_regrid.to_netcdf("ds_b_regrid.nc") + +# %% +# Regridding with CDAT +ds_mv1 = cdms2.open(fp_mv1) +ds_mv2 = cdms2.open(fp_mv2) + +mv1 = ds_mv1("variable_21") +mv2 = ds_mv2("variable_36") +mv_grid = mv1.getGrid() +mv2_reg = mv2.regrid(mv_grid, regridTool=regrid_tool, regridMethod=regrid_method) + +# Write out to netCDF for visualization +# f1 = cdms2.open("mv2_regrid.nc", "w") +# f1.write(mv2) +# f1.close() + +# %% +""" +1. Compare CDAT original data with regridded data + +Result: There is no difference between the original and regridded reference +data. +""" +# Test closeness (same) +np.testing.assert_allclose(mv2, mv2_reg, atol=0, rtol=0) + +mv2_filled = mv2.filled(np.nan) +mv2_reg_filled = mv2_reg.filled(np.nan) + +# Count of np.nan: 22170 vs. 22170 (same) +np.count_nonzero(np.isnan(mv2_filled.flatten())) == np.count_nonzero( + np.isnan(mv2_reg_filled.flatten()) +) +# Sum: -68837.01023016105 vs. -68837.01023016105 (same) +np.nansum(mv2_filled) == np.nansum(mv2_reg_filled) +# Mean: -6.342086809486 == -6.342086809486 (same) +np.nanmean(mv2_filled) == np.nanmean(mv2_reg_filled) + +# %% +""" +2. Compare regridded data between xCDAT and CDAT + +Result: The regridded reference data produced by xCDAT results in more `np.nan`, +which subsequently results in different np.nan locations, sum, and mean. + +In https://github.com/E3SM-Project/e3sm_diags/pull/794, I noted that there +seems tobe a minor difference in how the bilinear regridding works with xCDAT + xESMF vs. +CDAT + ESMF. +""" +# Test closeness (x and y nan location mismatch) +np.testing.assert_allclose(ds_b_regrid[var_key].values, mv2_reg, atol=0, rtol=0) + +# Count of np.nan: 23150 vs. 22170 (different) +np.count_nonzero(np.isnan(ds_b_regrid[var_key].values.flatten())) == np.count_nonzero( + np.isnan(mv2_reg_filled.flatten()) +) +# Sum: -77674.06388742107 vs. -68837.01023016105 (different) +np.nansum(ds_b_regrid[var_key].values) == np.nansum(mv2_reg_filled.data) +# Mean: -7.866524598685545 vs. -6.342086809486 (different) +np.nanmean(ds_b_regrid[var_key].values) == np.nanmean(mv2_reg_filled.data) + +# %% diff --git a/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/trefht_regrid_visual.py b/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/trefht_regrid_visual.py new file mode 100644 index 000000000..1915f9dac --- /dev/null +++ b/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/trefht_regrid_visual.py @@ -0,0 +1,65 @@ +"""This script compares the xCDAT and CDAT regridded reference data results. + +The `debug_TREFHT_diff.png` shows there there is a difference along the coastlines +of the continents. CDAT doesn't seem to actually regrid the data, which is +noted in `trefht_cdat_only_visual.py`. +""" +# %% +import os + +import numpy as np +import xarray as xr + +from auxiliary_tools.cdat_regression_testing.utils import get_image_diffs + +dir_path = "/global/u2/v/vo13/E3SM-Project/e3sm_diags/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug/" +fp_a = os.path.join(dir_path, "ds_b_regrid.nc") +fp_b = os.path.join(dir_path, "mv2_regrid.nc") + + +# %% +ds1 = xr.open_dataset(fp_a) +ds2 = xr.open_dataset(fp_b) + +var_key = "TREFHT" +ds2["TREFHT"] = ds2["variable_36"].copy() +ds2 = ds2.drop_vars("variable_36") + +# %% +try: + np.testing.assert_allclose(ds1[var_key], ds2[var_key]) +except AssertionError as e: + print(e) + +# %% +# Check the sum values -- close +# array(213927.85820162) +np.abs(ds1[var_key]).sum() + +# %% +# array(228804.60960445) +np.abs(ds2[var_key]).sum() + +# Check the mean values -- close +# array(-7.8665246) +ds1[var_key].mean() + +# array(-6.34208681) +ds2[var_key].mean() + + +# %% +# Check the plots and their diffs +root_dir = "auxiliary_tools/cdat_regression_testing/759-slice-flag" +actual_path = os.path.join(root_dir, "debug_TREFHT_actual.png") +expected_path = os.path.join(root_dir, "debug_TREFHT_expected.png") + +ax1 = ds1[var_key].plot() +ax1.figure.savefig(actual_path) + +# %% +ax2 = ds2[var_key].plot() +ax2.figure.savefig(expected_path) + +# %% +get_image_diffs(actual_path, expected_path) diff --git a/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug_TREFHT_actual.png b/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug_TREFHT_actual.png new file mode 100644 index 000000000..766063e03 Binary files /dev/null and b/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug_TREFHT_actual.png differ diff --git a/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug_TREFHT_diff.png b/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug_TREFHT_diff.png new file mode 100644 index 000000000..d9c2a4103 Binary files /dev/null and b/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug_TREFHT_diff.png differ diff --git a/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug_TREFHT_expected.png b/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug_TREFHT_expected.png new file mode 100644 index 000000000..ac1faf1a7 Binary files /dev/null and b/auxiliary_tools/cdat_regression_testing/759-slice-flag/debug_TREFHT_expected.png differ diff --git a/auxiliary_tools/cdat_regression_testing/759-slice-flag/ex1.py b/auxiliary_tools/cdat_regression_testing/759-slice-flag/ex1.py new file mode 100644 index 000000000..9e5b3d8cd --- /dev/null +++ b/auxiliary_tools/cdat_regression_testing/759-slice-flag/ex1.py @@ -0,0 +1,64 @@ +# %% +import os +import sys + +from e3sm_diags.parameter.core_parameter import CoreParameter +from e3sm_diags.run import runner + +param = CoreParameter() + +# Location of the data. +param.test_data_path = "/global/cfs/cdirs/e3sm/e3sm_diags/test_model_data_for_acme_diags/time-series/E3SM_v1" +param.reference_data_path = "/global/cfs/cdirs/e3sm/e3sm_diags/test_model_data_for_acme_diags/time-series/E3SM_v1" + +# Variables +# param.variables = ["PRECT"] + +# Set this parameter to True. +# By default, e3sm_diags expects the test data to be climo data. +param.test_timeseries_input = True +# Years to slice the test data, base this off the years in the filenames. +param.test_start_yr = "2011" +param.test_end_yr = "2013" + +# Set this parameter to True. +# By default, e3sm_diags expects the ref data to be climo data. +param.ref_timeseries_input = True +# Years to slice the ref data, base this off the years in the filenames +param.ref_start_yr = "1850" +param.ref_end_yr = "1852" + +# When running with time-series data, you don't need to specify the name of the data. +# But you should, otherwise nothing is displayed when the test/ref name is needed. +param.short_test_name = "historical_H1" +param.short_ref_name = "historical_H1" + +# This parameter modifies the software to accommodate model vs model runs. +# The default setting for run_type is 'model_vs_obs'. +param.run_type = "model_vs_model" +# Name of the folder where the results are stored. +# Change `prefix` to use your directory. +prefix = "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24" +param.results_dir = os.path.join(prefix, "759-slice-flag") + +# Below are more optional arguments. + +# What plotsets to run the diags on. +# If not defined, then all available sets are used. +param.sets = ["lat_lon"] +# What seasons to run the diags on. +# If not defined, diags are run on ['ANN', 'DJF', 'MAM', 'JJA', 'SON']. +param.seasons = ["ANN"] +# Title of the difference plots. +param.diff_title = "Model (2011-2013) - Model (1850-1852)" + +# For running with multiprocessing. +param.multiprocessing = True +# param.num_workers = 24 + +# %% +cfg_path = "/global/u2/v/vo13/E3SM-Project/e3sm_diags/auxiliary_tools/cdat_regression_testing/759-slice-flag/671-diags.cfg" +sys.argv.extend(["--diags", cfg_path]) +runner.run_diags([param]) + +# %% diff --git a/auxiliary_tools/cdat_regression_testing/759-slice-flag/regression-test-json.ipynb b/auxiliary_tools/cdat_regression_testing/759-slice-flag/regression-test-json.ipynb new file mode 100644 index 000000000..602d1120c --- /dev/null +++ b/auxiliary_tools/cdat_regression_testing/759-slice-flag/regression-test-json.ipynb @@ -0,0 +1,793 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# CDAT Migration Regression Testing Notebook (`.json` metrics)\n", + "\n", + "This notebook is used to perform regression testing between the development and\n", + "production versions of a diagnostic set.\n", + "\n", + "## How it works\n", + "\n", + "It compares the relative differences (%) between two sets of `.json` files in two\n", + "separate directories, one for the refactored code and the other for the `main` branch.\n", + "\n", + "It will display metrics values with relative differences >= 2%. Relative differences are used instead of absolute differences because:\n", + "\n", + "- Relative differences are in percentages, which shows the scale of the differences.\n", + "- Absolute differences are just a raw number that doesn't factor in\n", + " floating point size (e.g., 100.00 vs. 0.0001), which can be misleading.\n", + "\n", + "## How to use\n", + "\n", + "PREREQUISITE: The diagnostic set's metrics stored in `.json` files in two directories\n", + "(dev and `main` branches).\n", + "\n", + "1. Make a copy of this notebook under `auxiliary_tools/cdat_regression_testing/`.\n", + "2. Run `mamba create -n cdat_regression_test -y -c conda-forge \"python<3.12\" xarray netcdf4 dask pandas matplotlib-base ipykernel`\n", + "3. Run `mamba activate cdat_regression_test`\n", + "4. Update `DEV_PATH` and `MAIN_PATH` in the copy of your notebook.\n", + "5. Run all cells IN ORDER.\n", + "6. Review results for any outstanding differences (>= 2%).\n", + " - Debug these differences (e.g., bug in metrics functions, incorrect variable references, etc.)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup Code\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "ename": "OSError", + "evalue": "Number of files do not match at DEV_PATH and MAIN_PATH.", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mOSError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[1], line 29\u001b[0m\n\u001b[1;32m 26\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mIOError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNo files found at DEV_PATH and/or MAIN_PATH.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 28\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(DEV_GLOB) \u001b[38;5;241m!=\u001b[39m \u001b[38;5;28mlen\u001b[39m(MAIN_GLOB):\n\u001b[0;32m---> 29\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mIOError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNumber of files do not match at DEV_PATH and MAIN_PATH.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "\u001b[0;31mOSError\u001b[0m: Number of files do not match at DEV_PATH and MAIN_PATH." + ] + } + ], + "source": [ + "from typing import List\n", + "import glob\n", + "\n", + "import pandas as pd\n", + "\n", + "from auxiliary_tools.cdat_regression_testing.utils import (\n", + " get_rel_diffs,\n", + " get_num_metrics_above_diff_thres,\n", + " highlight_large_diffs,\n", + " sort_columns,\n", + " update_diffs_to_pct,\n", + " PERCENTAGE_COLUMNS,\n", + ")\n", + "\n", + "\n", + "SET_NAME = \"lat_lon\"\n", + "SET_DIR = \"759-slice-flag\"\n", + "\n", + "DEV_PATH = f\"/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/{SET_DIR}/{SET_NAME}/**\"\n", + "MAIN_PATH = f\"/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/old-runs/671-lat-lon/main/{SET_NAME}/**\"\n", + "\n", + "DEV_GLOB = sorted(glob.glob(DEV_PATH + \"/*.json\"))\n", + "MAIN_GLOB = sorted(glob.glob(MAIN_PATH + \"/*.json\"))\n", + "\n", + "if len(DEV_GLOB) == 0 or len(MAIN_GLOB) == 0:\n", + " raise IOError(\"No files found at DEV_PATH and/or MAIN_PATH.\")\n", + "\n", + "if len(DEV_GLOB) != len(MAIN_GLOB):\n", + " raise IOError(\"Number of files do not match at DEV_PATH and MAIN_PATH.\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "dev_filenames = [filepaths.split(\"/\")[-1] for filepaths in DEV_GLOB]\n", + "main_filenames = [filepaths.split(\"/\")[-1] for filepaths in MAIN_GLOB]" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'-ERFtot-ANN-global.json'}" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "set(dev_filenames) ^ set(main_filenames)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "def get_metrics(filepaths: List[str]) -> pd.DataFrame:\n", + " \"\"\"Get the metrics using a glob of `.json` metric files in a directory.\n", + "\n", + " Parameters\n", + " ----------\n", + " filepaths : List[str]\n", + " The filepaths for metrics `.json` files.\n", + "\n", + " Returns\n", + " -------\n", + " pd.DataFrame\n", + " The DataFrame containing the metrics for all of the variables in\n", + " the results directory.\n", + " \"\"\"\n", + " metrics = []\n", + "\n", + " for filepath in filepaths:\n", + " df = pd.read_json(filepath)\n", + "\n", + " filename = filepath.split(\"/\")[-1]\n", + " var_key = filename.split(\"-\")[1]\n", + " region = filename.split(\"-\")[-1].replace(\".json\", \"\")\n", + "\n", + " # Add the variable key to the MultiIndex and update the index\n", + " # before stacking to make the DataFrame easier to parse.\n", + " multiindex = pd.MultiIndex.from_product([[var_key], [region], [*df.index]])\n", + " df = df.set_index(multiindex)\n", + " df.stack()\n", + "\n", + " metrics.append(df)\n", + "\n", + " df_final = pd.concat(metrics)\n", + "\n", + " # Reorder columns and drop \"unit\" column (string dtype breaks Pandas\n", + " # arithmetic).\n", + " df_final = df_final[[\"test\", \"ref\", \"test_regrid\", \"ref_regrid\", \"diff\", \"misc\"]]\n", + "\n", + " return df_final" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Get the metrics for the development and `main` branches and their differences.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "df_metrics_dev = get_metrics(DEV_GLOB).drop(\"ERFtot\", level=0)\n", + "df_metrics_main = get_metrics(MAIN_GLOB)\n", + "df_metrics_diffs = get_rel_diffs(df_metrics_dev, df_metrics_main)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Filter differences to those above maximum threshold (2%).\n", + "\n", + "All values below maximum threshold will be labeled as `NaN`.\n", + "\n", + "- **If all cells in a row are NaN (< 2%)**, the entire row is dropped to make the results easier to parse.\n", + "- Any remaining NaN cells are below < 2% difference and **should be ignored**.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "df_metrics_diffs_thres = df_metrics_diffs[df_metrics_diffs >= 0.02]\n", + "df_metrics_diffs_thres = df_metrics_diffs_thres.dropna(\n", + " axis=0, how=\"all\", ignore_index=False\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
test DIFF (%)ref DIFF (%)test_regrid DIFF (%)ref_regrid DIFF (%)diff DIFF (%)misc DIFF (%)
SSTglobalminNaNNaNNaN0.5131070.203545NaN
TREFHTlandmeanNaNNaNNaN0.115921NaNNaN
stdNaNNaNNaN0.028019NaNNaN
\n", + "
" + ], + "text/plain": [ + " test DIFF (%) ref DIFF (%) test_regrid DIFF (%) \\\n", + "SST global min NaN NaN NaN \n", + "TREFHT land mean NaN NaN NaN \n", + " std NaN NaN NaN \n", + "\n", + " ref_regrid DIFF (%) diff DIFF (%) misc DIFF (%) \n", + "SST global min 0.513107 0.203545 NaN \n", + "TREFHT land mean 0.115921 NaN NaN \n", + " std 0.028019 NaN NaN " + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_metrics_diffs_thres" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Combine all DataFrames to get the final result.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "df_metrics_all = pd.concat(\n", + " [df_metrics_dev.add_suffix(\"_dev\"), df_metrics_main.add_suffix(\"_main\")],\n", + " axis=1,\n", + " join=\"outer\",\n", + ")\n", + "\n", + "df_final = df_metrics_all.join(df_metrics_diffs_thres, how=\"inner\")\n", + "df_final = sort_columns(df_final)\n", + "df_final = update_diffs_to_pct(df_final)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
test_devtest_maintest DIFF (%)ref_devref_mainref DIFF (%)test_regrid_devtest_regrid_maintest_regrid DIFF (%)ref_regrid_devref_regrid_mainref_regrid DIFF (%)misc_devmisc_mainmisc DIFF (%)
SSTglobalmin-1.788055-1.788055NaN-1.676941-1.676941NaN-1.788055-1.788055NaN-1.108276-1.67694151.31%NaNNaNNaN
TREFHTlandmean9.1145729.114572NaN7.9579177.957917NaN9.1145729.114572NaN7.1312577.95791711.59%NaNNaNNaN
stdNaNNaNNaNNaNNaNNaN17.94774317.947743NaN18.71867518.1941962.80%NaNNaNNaN
\n", + "
" + ], + "text/plain": [ + " test_dev test_main test DIFF (%) ref_dev ref_main \\\n", + "SST global min -1.788055 -1.788055 NaN -1.676941 -1.676941 \n", + "TREFHT land mean 9.114572 9.114572 NaN 7.957917 7.957917 \n", + " std NaN NaN NaN NaN NaN \n", + "\n", + " ref DIFF (%) test_regrid_dev test_regrid_main \\\n", + "SST global min NaN -1.788055 -1.788055 \n", + "TREFHT land mean NaN 9.114572 9.114572 \n", + " std NaN 17.947743 17.947743 \n", + "\n", + " test_regrid DIFF (%) ref_regrid_dev ref_regrid_main \\\n", + "SST global min NaN -1.108276 -1.676941 \n", + "TREFHT land mean NaN 7.131257 7.957917 \n", + " std NaN 18.718675 18.194196 \n", + "\n", + " ref_regrid DIFF (%) misc_dev misc_main misc DIFF (%) \n", + "SST global min 51.31% NaN NaN NaN \n", + "TREFHT land mean 11.59% NaN NaN NaN \n", + " std 2.80% NaN NaN NaN " + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_final" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Review variables and metrics above difference threshold.\n", + "\n", + "- Red cells are differences >= 2%\n", + "- `nan` cells are differences < 2% and **should be ignored**\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "* Related variables ['SST', 'TREFHT']\n", + "* Number of metrics above 2% max threshold: 3 / 96\n" + ] + } + ], + "source": [ + "df_final_adj = df_final.reset_index(names=[\"var_key\", \"region\", \"metric\"])\n", + "get_num_metrics_above_diff_thres(df_metrics_all, df_final_adj)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
var_keyregionmetrictest_devtest_maintest DIFF (%)ref_devref_mainref DIFF (%)test_regrid_devtest_regrid_maintest_regrid DIFF (%)ref_regrid_devref_regrid_mainref_regrid DIFF (%)misc_devmisc_mainmisc DIFF (%)
0SSTglobalmin-1.788055-1.788055NaN-1.676941-1.676941NaN-1.788055-1.788055NaN-1.108276-1.67694151.31%NaNNaNNaN
1TREFHTlandmean9.1145729.114572NaN7.9579177.957917NaN9.1145729.114572NaN7.1312577.95791711.59%NaNNaNNaN
2TREFHTlandstdNaNNaNNaNNaNNaNNaN17.94774317.947743NaN18.71867518.1941962.80%NaNNaNNaN
\n", + "
" + ], + "text/plain": [ + " var_key region metric test_dev test_main test DIFF (%) ref_dev \\\n", + "0 SST global min -1.788055 -1.788055 NaN -1.676941 \n", + "1 TREFHT land mean 9.114572 9.114572 NaN 7.957917 \n", + "2 TREFHT land std NaN NaN NaN NaN \n", + "\n", + " ref_main ref DIFF (%) test_regrid_dev test_regrid_main \\\n", + "0 -1.676941 NaN -1.788055 -1.788055 \n", + "1 7.957917 NaN 9.114572 9.114572 \n", + "2 NaN NaN 17.947743 17.947743 \n", + "\n", + " test_regrid DIFF (%) ref_regrid_dev ref_regrid_main ref_regrid DIFF (%) \\\n", + "0 NaN -1.108276 -1.676941 51.31% \n", + "1 NaN 7.131257 7.957917 11.59% \n", + "2 NaN 18.718675 18.194196 2.80% \n", + "\n", + " misc_dev misc_main misc DIFF (%) \n", + "0 NaN NaN NaN \n", + "1 NaN NaN NaN \n", + "2 NaN NaN NaN " + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_final_adj" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
 var_keyregionmetrictest_devtest_maintest DIFF (%)ref_devref_mainref DIFF (%)test_regrid_devtest_regrid_maintest_regrid DIFF (%)ref_regrid_devref_regrid_mainref_regrid DIFF (%)misc_devmisc_mainmisc DIFF (%)
0SSTglobalmin-1.788055-1.788055nan-1.676941-1.676941nan-1.788055-1.788055nan-1.108276-1.67694151.31%nannannan
1TREFHTlandmean9.1145729.114572nan7.9579177.957917nan9.1145729.114572nan7.1312577.95791711.59%nannannan
2TREFHTlandstdnannannannannannan17.94774317.947743nan18.71867518.1941962.80%nannannan
\n" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "highlight_large_diffs(df_final_adj)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Results\n", + "\n", + "- The last remaining large diff is the regridded `TREFHT` `\"land\"` mean. This will be addressed in a separate PR.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "cdat_regression_test", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/auxiliary_tools/cdat_regression_testing/759-slice-flag/regression-test-netcdf.ipynb b/auxiliary_tools/cdat_regression_testing/759-slice-flag/regression-test-netcdf.ipynb new file mode 100644 index 000000000..cd7a385f2 --- /dev/null +++ b/auxiliary_tools/cdat_regression_testing/759-slice-flag/regression-test-netcdf.ipynb @@ -0,0 +1,1565 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# CDAT Migration Regression Testing Notebook (`.nc` files)\n", + "\n", + "This notebook is used to perform regression testing between the development and\n", + "production versions of a diagnostic set.\n", + "\n", + "## How it works\n", + "\n", + "It compares the relative differences (%) between ref and test variables between\n", + "the dev and `main` branches.\n", + "\n", + "## How to use\n", + "\n", + "PREREQUISITE: The diagnostic set's netCDF stored in `.json` files in two directories\n", + "(dev and `main` branches).\n", + "\n", + "1. Make a copy of this notebook under `auxiliary_tools/cdat_regression_testing/`.\n", + "2. Run `mamba create -n cdat_regression_test -y -c conda-forge \"python<3.12\" xarray netcdf4 dask pandas matplotlib-base ipykernel`\n", + "3. Run `mamba activate cdat_regression_test`\n", + "4. Update `SET_DIR` and `SET_NAME` in the copy of your notebook.\n", + "5. Run all cells IN ORDER.\n", + "6. Review results for any outstanding differences (>=1e-5 relative tolerance).\n", + " - Debug these differences (e.g., bug in metrics functions, incorrect variable references, etc.)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup Code\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from collections import defaultdict\n", + "import glob\n", + "from typing import Tuple\n", + "\n", + "import numpy as np\n", + "import xarray as xr\n", + "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", + "\n", + "\n", + "# TODO: Update SET_NAME and SET_DIR\n", + "SET_NAME = \"area_mean_time_series\"\n", + "SET_DIR = \"662-area-mean-time-series\"\n", + "\n", + "DEV_PATH = f\"/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/{SET_DIR}/{SET_NAME}/**\"\n", + "DEV_GLOB = sorted(glob.glob(DEV_PATH + \"/*.nc\"))\n", + "DEV_NUM_FILES = len(DEV_GLOB)\n", + "\n", + "MAIN_PATH = f\"/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/{SET_NAME}/**\"\n", + "MAIN_GLOB = sorted(glob.glob(MAIN_PATH + \"/*.nc\"))\n", + "MAIN_NUM_FILES = len(MAIN_GLOB)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "def _check_if_files_found():\n", + " if DEV_NUM_FILES == 0 or MAIN_NUM_FILES == 0:\n", + " raise IOError(\n", + " \"No files found at DEV_PATH and/or MAIN_PATH. \"\n", + " f\"Please check {DEV_PATH} and {MAIN_PATH}.\"\n", + " )\n", + "\n", + "\n", + "def _check_if_matching_filecount():\n", + " if DEV_NUM_FILES != MAIN_NUM_FILES:\n", + " raise IOError(\n", + " \"Number of files do not match at DEV_PATH and MAIN_PATH \"\n", + " f\"({DEV_NUM_FILES} vs. {MAIN_NUM_FILES}).\"\n", + " )\n", + "\n", + " print(f\"Matching file count ({DEV_NUM_FILES} and {MAIN_NUM_FILES}).\")\n", + "\n", + "\n", + "def _check_if_missing_files():\n", + " missing_count = 0\n", + "\n", + " for fp_main in MAIN_GLOB:\n", + " fp_dev = fp_main.replace(SET_DIR, \"main-area-mean-time-series\")\n", + "\n", + " if fp_dev not in MAIN_GLOB:\n", + " print(f\"No production file found to compare with {fp_dev}!\")\n", + " missing_count += 1\n", + "\n", + " for fp_dev in DEV_GLOB:\n", + " fp_main = fp_main.replace(\"main-area-mean-time-series\", SET_DIR)\n", + "\n", + " if fp_main not in DEV_GLOB:\n", + " print(f\"No development file found to compare with {fp_main}!\")\n", + " missing_count += 1\n", + "\n", + " print(f\"Number of files missing: {missing_count}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def _get_relative_diffs():\n", + " # We are mainly focusing on relative tolerance here (in percentage terms).\n", + " atol = 0\n", + " rtol = 1e-5\n", + "\n", + " for fp_main in MAIN_GLOB:\n", + " if \"test.nc\" in fp_main or \"ref.nc\" in fp_main:\n", + " fp_dev = fp_main.replace(\"main-area-mean-time-series\", SET_DIR)\n", + "\n", + " print(\"Comparing:\")\n", + " print(f\" * {fp_dev}\")\n", + " print(f\" * {fp_main}\")\n", + "\n", + " ds1 = xr.open_dataset(fp_dev)\n", + " ds2 = xr.open_dataset(fp_main)\n", + "\n", + " var_key = fp_main.split(\"-\")[-3]\n", + " # for 3d vars such as T-200\n", + " var_key.isdigit()\n", + " if var_key.isdigit():\n", + " var_key = fp_main.split(\"-\")[-4]\n", + "\n", + " print(f\" * var_key: {var_key}\")\n", + "\n", + " dev_data = _get_var_data(ds1, var_key)\n", + " main_data = _get_var_data(ds2, var_key)\n", + "\n", + " if dev_data is None or main_data is None:\n", + " print(\" * Could not find variable key in the dataset(s)\")\n", + " continue\n", + "\n", + " try:\n", + " np.testing.assert_allclose(\n", + " dev_data,\n", + " main_data,\n", + " atol=atol,\n", + " rtol=rtol,\n", + " )\n", + " except (KeyError, AssertionError) as e:\n", + " print(f\" {e}\")\n", + " else:\n", + " print(f\" * All close and within relative tolerance ({rtol})\")\n", + "\n", + "\n", + "def _get_var_data(ds: xr.Dataset, var_key: str) -> np.ndarray:\n", + " \"\"\"Get the variable data using a list of matching keys.\n", + "\n", + " The `main` branch saves the dataset using the original variable name,\n", + " while the dev branch saves the variable with the derived variable name.\n", + " The dev branch is performing the expected behavior here.\n", + "\n", + " Parameters\n", + " ----------\n", + " ds : xr.Dataset\n", + " _description_\n", + " var_key : str\n", + " _description_\n", + "\n", + " Returns\n", + " -------\n", + " np.ndarray\n", + " _description_\n", + " \"\"\"\n", + "\n", + " data = None\n", + "\n", + " var_keys = DERIVED_VARIABLES[var_key].keys()\n", + " var_keys = [var_key] + list(sum(var_keys, ()))\n", + "\n", + " for key in var_keys:\n", + " if key in ds.data_vars.keys():\n", + " data = ds[key].values\n", + " break\n", + "\n", + " return data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Check for matching and equal number of files\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "_check_if_files_found()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Matching file count (72 and 72).\n" + ] + } + ], + "source": [ + "_check_if_matching_filecount()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Number of files missing: 0\n" + ] + } + ], + "source": [ + "_check_if_missing_files()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2 Compare the netCDF files between branches\n", + "\n", + "- Compare \"ref\" and \"test\" files\n", + "- \"diff\" files are ignored because getting relative diffs for these does not make sense (relative diff will be above tolerance)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/FLUT/FLUT-20N50N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/FLUT/FLUT-20N50N_test.nc\n", + "var_key FLUT\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/FLUT/FLUT-20S20N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/FLUT/FLUT-20S20N_test.nc\n", + "var_key FLUT\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/FLUT/FLUT-50N90N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/FLUT/FLUT-50N90N_test.nc\n", + "var_key FLUT\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/FLUT/FLUT-50S20S_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/FLUT/FLUT-50S20S_test.nc\n", + "var_key FLUT\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/FLUT/FLUT-90S50S_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/FLUT/FLUT-90S50S_test.nc\n", + "var_key FLUT\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/FLUT/FLUT-global_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/FLUT/FLUT-global_test.nc\n", + "var_key FLUT\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/FLUT/FLUT-land_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/FLUT/FLUT-land_test.nc\n", + "var_key FLUT\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/FLUT/FLUT-ocean_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/FLUT/FLUT-ocean_test.nc\n", + "var_key FLUT\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/FSNTOA/FSNTOA-20N50N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/FSNTOA/FSNTOA-20N50N_test.nc\n", + "var_key FSNTOA\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/FSNTOA/FSNTOA-20S20N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/FSNTOA/FSNTOA-20S20N_test.nc\n", + "var_key FSNTOA\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/FSNTOA/FSNTOA-50N90N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/FSNTOA/FSNTOA-50N90N_test.nc\n", + "var_key FSNTOA\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/FSNTOA/FSNTOA-50S20S_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/FSNTOA/FSNTOA-50S20S_test.nc\n", + "var_key FSNTOA\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/FSNTOA/FSNTOA-90S50S_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/FSNTOA/FSNTOA-90S50S_test.nc\n", + "var_key FSNTOA\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/FSNTOA/FSNTOA-global_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/FSNTOA/FSNTOA-global_test.nc\n", + "var_key FSNTOA\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/FSNTOA/FSNTOA-land_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/FSNTOA/FSNTOA-land_test.nc\n", + "var_key FSNTOA\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/FSNTOA/FSNTOA-ocean_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/FSNTOA/FSNTOA-ocean_test.nc\n", + "var_key FSNTOA\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/LHFLX/LHFLX-20N50N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/LHFLX/LHFLX-20N50N_test.nc\n", + "var_key LHFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/LHFLX/LHFLX-20S20N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/LHFLX/LHFLX-20S20N_test.nc\n", + "var_key LHFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/LHFLX/LHFLX-50N90N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/LHFLX/LHFLX-50N90N_test.nc\n", + "var_key LHFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/LHFLX/LHFLX-50S20S_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/LHFLX/LHFLX-50S20S_test.nc\n", + "var_key LHFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/LHFLX/LHFLX-90S50S_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/LHFLX/LHFLX-90S50S_test.nc\n", + "var_key LHFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/LHFLX/LHFLX-global_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/LHFLX/LHFLX-global_test.nc\n", + "var_key LHFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/LHFLX/LHFLX-land_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/LHFLX/LHFLX-land_test.nc\n", + "var_key LHFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/LHFLX/LHFLX-ocean_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/LHFLX/LHFLX-ocean_test.nc\n", + "var_key LHFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/LWCF/LWCF-20N50N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/LWCF/LWCF-20N50N_test.nc\n", + "var_key LWCF\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/LWCF/LWCF-20S20N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/LWCF/LWCF-20S20N_test.nc\n", + "var_key LWCF\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/LWCF/LWCF-50N90N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/LWCF/LWCF-50N90N_test.nc\n", + "var_key LWCF\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/LWCF/LWCF-50S20S_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/LWCF/LWCF-50S20S_test.nc\n", + "var_key LWCF\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/LWCF/LWCF-90S50S_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/LWCF/LWCF-90S50S_test.nc\n", + "var_key LWCF\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/LWCF/LWCF-global_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/LWCF/LWCF-global_test.nc\n", + "var_key LWCF\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/LWCF/LWCF-land_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/LWCF/LWCF-land_test.nc\n", + "var_key LWCF\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/LWCF/LWCF-ocean_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/LWCF/LWCF-ocean_test.nc\n", + "var_key LWCF\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/PRECT/PRECT-20N50N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/PRECT/PRECT-20N50N_test.nc\n", + "var_key PRECT\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/PRECT/PRECT-20S20N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/PRECT/PRECT-20S20N_test.nc\n", + "var_key PRECT\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/PRECT/PRECT-50N90N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/PRECT/PRECT-50N90N_test.nc\n", + "var_key PRECT\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/PRECT/PRECT-50S20S_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/PRECT/PRECT-50S20S_test.nc\n", + "var_key PRECT\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/PRECT/PRECT-90S50S_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/PRECT/PRECT-90S50S_test.nc\n", + "var_key PRECT\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/PRECT/PRECT-global_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/PRECT/PRECT-global_test.nc\n", + "var_key PRECT\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/PRECT/PRECT-land_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/PRECT/PRECT-land_test.nc\n", + "var_key PRECT\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/PRECT/PRECT-ocean_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/PRECT/PRECT-ocean_test.nc\n", + "var_key PRECT\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/QFLX/QFLX-20N50N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/QFLX/QFLX-20N50N_test.nc\n", + "var_key QFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/QFLX/QFLX-20S20N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/QFLX/QFLX-20S20N_test.nc\n", + "var_key QFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/QFLX/QFLX-50N90N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/QFLX/QFLX-50N90N_test.nc\n", + "var_key QFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/QFLX/QFLX-50S20S_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/QFLX/QFLX-50S20S_test.nc\n", + "var_key QFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/QFLX/QFLX-90S50S_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/QFLX/QFLX-90S50S_test.nc\n", + "var_key QFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/QFLX/QFLX-global_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/QFLX/QFLX-global_test.nc\n", + "var_key QFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/QFLX/QFLX-land_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/QFLX/QFLX-land_test.nc\n", + "var_key QFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/QFLX/QFLX-ocean_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/QFLX/QFLX-ocean_test.nc\n", + "var_key QFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/SHFLX/SHFLX-20N50N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/SHFLX/SHFLX-20N50N_test.nc\n", + "var_key SHFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/SHFLX/SHFLX-20S20N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/SHFLX/SHFLX-20S20N_test.nc\n", + "var_key SHFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/SHFLX/SHFLX-50N90N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/SHFLX/SHFLX-50N90N_test.nc\n", + "var_key SHFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/SHFLX/SHFLX-50S20S_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/SHFLX/SHFLX-50S20S_test.nc\n", + "var_key SHFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/SHFLX/SHFLX-90S50S_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/SHFLX/SHFLX-90S50S_test.nc\n", + "var_key SHFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/SHFLX/SHFLX-global_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/SHFLX/SHFLX-global_test.nc\n", + "var_key SHFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/SHFLX/SHFLX-land_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/SHFLX/SHFLX-land_test.nc\n", + "var_key SHFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/SHFLX/SHFLX-ocean_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/SHFLX/SHFLX-ocean_test.nc\n", + "var_key SHFLX\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/SWCF/SWCF-20N50N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/SWCF/SWCF-20N50N_test.nc\n", + "var_key SWCF\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/SWCF/SWCF-20S20N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/SWCF/SWCF-20S20N_test.nc\n", + "var_key SWCF\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/SWCF/SWCF-50N90N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/SWCF/SWCF-50N90N_test.nc\n", + "var_key SWCF\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/SWCF/SWCF-50S20S_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/SWCF/SWCF-50S20S_test.nc\n", + "var_key SWCF\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/SWCF/SWCF-90S50S_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/SWCF/SWCF-90S50S_test.nc\n", + "var_key SWCF\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/SWCF/SWCF-global_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/SWCF/SWCF-global_test.nc\n", + "var_key SWCF\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/SWCF/SWCF-land_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/SWCF/SWCF-land_test.nc\n", + "var_key SWCF\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/SWCF/SWCF-ocean_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/SWCF/SWCF-ocean_test.nc\n", + "var_key SWCF\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/TREFHT/TREFHT-20N50N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/TREFHT/TREFHT-20N50N_test.nc\n", + "var_key TREFHT\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/TREFHT/TREFHT-20S20N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/TREFHT/TREFHT-20S20N_test.nc\n", + "var_key TREFHT\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/TREFHT/TREFHT-50N90N_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/TREFHT/TREFHT-50N90N_test.nc\n", + "var_key TREFHT\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/TREFHT/TREFHT-50S20S_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/TREFHT/TREFHT-50S20S_test.nc\n", + "var_key TREFHT\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/TREFHT/TREFHT-90S50S_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/TREFHT/TREFHT-90S50S_test.nc\n", + "var_key TREFHT\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/TREFHT/TREFHT-global_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/TREFHT/TREFHT-global_test.nc\n", + "var_key TREFHT\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/TREFHT/TREFHT-land_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/TREFHT/TREFHT-land_test.nc\n", + "var_key TREFHT\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/TREFHT/TREFHT-ocean_test.nc \n", + " /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/TREFHT/TREFHT-ocean_test.nc\n", + "var_key TREFHT\n", + " * All close and within relative tolerance (1e-05)\n" + ] + } + ], + "source": [ + "# We are mainly focusing on relative tolerance here (in percentage terms).\n", + "atol = 0\n", + "rtol = 1e-5\n", + "\n", + "count_mismatch = 0\n", + "\n", + "for filepath_dev in DEV_GLOB:\n", + " if \"test.nc\" in filepath_dev or \"ref.nc\" in filepath_dev:\n", + " filepath_main = filepath_dev.replace(SET_DIR, \"main-area-mean-time-series\")\n", + " print(\"Comparing:\")\n", + " print(filepath_dev, \"\\n\", filepath_main)\n", + " ds1 = xr.open_dataset(filepath_dev)\n", + " ds2 = xr.open_dataset(filepath_main)\n", + "\n", + " var_key = filepath_dev.split(\"/\")[-2]\n", + " # for 3d vars such as T-200\n", + " var_key.isdigit()\n", + " if var_key.isdigit():\n", + " var_key = filepath_dev.split(\"-\")[-4]\n", + "\n", + " print(\"var_key\", var_key)\n", + " try:\n", + " np.testing.assert_allclose(\n", + " ds1[var_key].values,\n", + " ds2[var_key].values,\n", + " atol=atol,\n", + " rtol=rtol,\n", + " )\n", + " except AssertionError as e:\n", + " print(e)\n", + " count_mismatch += 1\n", + " else:\n", + " print(f\" * All close and within relative tolerance ({rtol})\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "72" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "count_mismatch" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "fp1 = \"/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/662-area-mean-time-series/area_mean_time_series/FLUT/FLUT-land_test.nc\"\n", + "fp2 = \"/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/main-area-mean-time-series/area_mean_time_series/FLUT/FLUT-land_test.nc\"" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "ds_fp1 = xr.open_dataset(fp1)\n", + "ds_fp2 = xr.open_dataset(fp2)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 160B\n",
+       "Dimensions:  (time: 10)\n",
+       "Coordinates:\n",
+       "  * time     (time) object 80B 0051-01-01 00:00:00 ... 0060-01-01 00:00:00\n",
+       "Data variables:\n",
+       "    FLUT     (time) float64 80B ...
" + ], + "text/plain": [ + " Size: 160B\n", + "Dimensions: (time: 10)\n", + "Coordinates:\n", + " * time (time) object 80B 0051-01-01 00:00:00 ... 0060-01-01 00:00:00\n", + "Data variables:\n", + " FLUT (time) float64 80B ..." + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds_fp1" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
<xarray.Dataset> Size: 320B\n",
+       "Dimensions:      (time: 10, bound: 2)\n",
+       "Coordinates:\n",
+       "  * time         (time) object 80B 0051-07-02 12:00:00 ... 0060-07-02 12:00:00\n",
+       "Dimensions without coordinates: bound\n",
+       "Data variables:\n",
+       "    bounds_time  (time, bound) object 160B ...\n",
+       "    FLUT         (time) float64 80B ...\n",
+       "Attributes:\n",
+       "    Conventions:  CF-1.0
" + ], + "text/plain": [ + " Size: 320B\n", + "Dimensions: (time: 10, bound: 2)\n", + "Coordinates:\n", + " * time (time) object 80B 0051-07-02 12:00:00 ... 0060-07-02 12:00:00\n", + "Dimensions without coordinates: bound\n", + "Data variables:\n", + " bounds_time (time, bound) object 160B ...\n", + " FLUT (time) float64 80B ...\n", + "Attributes:\n", + " Conventions: CF-1.0" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ds_fp2" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Results\n", + "\n", + "All files are within rtol 1e-5, so the changes should be good to go.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "cdat_regression_test", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/e3sm_diags/driver/utils/regrid.py b/e3sm_diags/driver/utils/regrid.py index 4eef030ae..64bad7875 100644 --- a/e3sm_diags/driver/utils/regrid.py +++ b/e3sm_diags/driver/utils/regrid.py @@ -91,8 +91,8 @@ def subset_and_align_datasets( # Subset on a specific region. if "global" not in region: - ds_test_new = _subset_on_region(ds_test, var_key, region) - ds_ref_new = _subset_on_region(ds_ref, var_key, region) + ds_test_new = _subset_on_region(ds_test_new, var_key, region) + ds_ref_new = _subset_on_region(ds_ref_new, var_key, region) ds_test_regrid, ds_ref_regrid = align_grids_to_lower_res( ds_test_new, @@ -297,20 +297,21 @@ def _subset_on_region(ds: xr.Dataset, var_key: str, region: str) -> xr.Dataset: Replaces `e3sm_diags.utils.general.select_region`. """ specs = REGION_SPECS[region] - lat, lon = specs.get("lat"), specs.get("lon") # type: ignore + ds_new = ds.copy() + if lat is not None: lat_dim = xc.get_dim_keys(ds[var_key], axis="Y") - ds = ds.sortby(lat_dim) - ds = ds.sel({f"{lat_dim}": slice(*lat)}) + ds_new = ds_new.sortby(lat_dim) + ds_new = ds_new.sel({f"{lat_dim}": slice(*lat)}) if lon is not None: lon_dim = xc.get_dim_keys(ds[var_key], axis="X") - ds = ds.sortby(lon_dim) - ds = ds.sel({f"{lon_dim}": slice(*lon)}) + ds_new = ds_new.sortby(lon_dim) + ds_new = ds_new.sel({f"{lon_dim}": slice(*lon)}) - return ds + return ds_new def _subset_on_arm_coord(ds: xr.Dataset, var_key: str, arm_site: str):