From 889f68305933166e6c8d77a34b4b2f93f26f5c84 Mon Sep 17 00:00:00 2001 From: Wei-Ting Hung <107704243+angehung5@users.noreply.github.com> Date: Tue, 3 Sep 2024 14:04:25 -0400 Subject: [PATCH 01/10] Delete src/rave_emission_cal.py --- src/rave_emission_cal.py | 155 --------------------------------------- 1 file changed, 155 deletions(-) delete mode 100755 src/rave_emission_cal.py diff --git a/src/rave_emission_cal.py b/src/rave_emission_cal.py deleted file mode 100755 index 6c440ca..0000000 --- a/src/rave_emission_cal.py +++ /dev/null @@ -1,155 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Author: whung - -This script is used to generate RAVE fire emission estimations based on Li et al. (2022). - -Li, F., Zhang, X., Kondragunta, S., Lu, X., Csiszar, I., Schmidt, C. C., 2022. Hourly biomass burning emissions product from blended geostationary and polar-orbiting satellites for air quality forecasting applications. Remote Sens. Environ. 281, 113237, https://doi.org/10.1016/j.rse.2022.113237. - -NOTE: THIS SCRIPT DOES NOT INCLUDED IN THE FIRE MODEL WORKFLOW. -""" - -from netCDF4 import Dataset -import pandas as pd -import numpy as np -import os - -import warnings -warnings.simplefilter(action='ignore') - - - -'''Settings''' -namelist = pd.read_csv('./input/namelist', header=None, delimiter='=') -namelist = namelist[1] - -time_start = str(namelist[4].replace(' ', '')) # yyyymmddHHMM -path_input = './input/'+time_start -path_output = './output/'+time_start -path_rave = str(namelist[21].replace(' ', '')) - -lat_lim = [float(namelist[7]), float(namelist[8])] -lon_lim = [float(namelist[9]), float(namelist[10])] - -## constants -f_scale = 1 # FRP scale factor -factor_FRE = 3600 # the ratio of hourly FRE to 5-min-avg FRP -fc = 0.368 # FRE biomass combustion factor (kg MJ-1) - - -def emi_estimator(fre, vtype): - DM = fre * fc # hourly consumed dry matter (kg) - - EF = np.zeros(DM.shape) # PM2.5 emission factor (g kg-1) - EF[vtype==1] = 12.8 # forest - EF[(vtype>=2) & (vtype<=4)] = 7.17 # savanna, shrubland, grassland - EF[vtype==5] = 6.26 # cropland - - M = (DM * EF) / 1000 # total mass of PM2.5 emission (kg) - - return M - - - -'''Land cover''' -## domain -readin = Dataset(path_input+'/rave.frp.conus.'+time_start+'.nc') -LAT = readin['grid_lat'][:] -LON = readin['grid_lon'][:] -readin.close() - - -## RAVE land cover -date = time_start[:8] -hour = time_start[8:10] - -#fname = 'Hourly_Emissions_3km_'+date+'0000_'+date+'2300.nc' -fname = 'RAVE-HrlyEmiss-3km_v1r3_blend_s'+date+hour+'0000' -f_rave = [f for f in os.listdir(path_rave) if fname in f][0] -f_rave = path_rave + '/' + f_rave - -readin = Dataset(f_rave) -yt = np.flip(readin['grid_latt'][:, 0]) -xt = readin['grid_lont'][0, :] -yt = np.round(yt, 3) -xt = np.round(xt, 3) -index1 = np.squeeze(np.argwhere((yt>=lat_lim[0]) & (yt<=lat_lim[1]))) -index2 = np.squeeze(np.argwhere((xt>=lon_lim[0]) & (xt<=lon_lim[1]))) - -VTYPE = np.squeeze(readin['land_cover'][:]) -VTYPE = np.flipud(VTYPE) - -if (index1[0] == 0) & (index2[0] == 0): - VTYPE = VTYPE[index1[0]:index1[-1]+2, index2[0]:index2[-1]+2] -elif index1[0] == 0: - VTYPE = VTYPE[index1[0]:index1[-1]+2, index2[0]-1:index2[-1]+2] -elif index2[0] == 0: - VTYPE = VTYPE[index1[0]-1:index1[-1]+2, index2[0]:index2[-1]+2] -else: - VTYPE = VTYPE[index1[0]-1:index1[-1]+2, index2[0]-1:index2[-1]+2] - -readin.close() -del [readin, yt, xt, index1, index2] - -#print(LAT.shape, LON.shape, VTYPE.shape) - - - -'''Fire emission estiamtion''' -filename = [file for file in os.listdir(path_output) if ('fire.pred.conus' in file) and (file.endswith('.nc'))] -filename = np.sort(filename) - -NN = len(filename) - -for i in np.arange(NN): - f_time = int(filename[i][30:32]) - f_input = path_output+'/'+filename[i] - f_output = path_output+'/'+(filename[i].replace('pred', 'emi')) - - if f_scale != 1: - f_output = f_output.replace('.nc', '.scale'+str(f_scale)+'.nc') - - #print(f_time, f_input, f_output) - - readin = Dataset(f_input) - FRP = readin['grid_predic'][0, :, :] * f_scale - FRP_SD = np.ones(FRP.shape) - FRE = FRP * factor_FRE - EMI = emi_estimator(FRE, VTYPE) - readin.close() - - print(LAT.shape, LON.shape) - print(FRP.shape, np.min(FRP[FRP!=0]), np.max(FRP[FRP!=0])) - print(FRE.shape, np.min(FRE[FRE!=0]), np.max(FRE[FRE!=0])) - print(EMI.shape, np.min(EMI[EMI!=0]), np.max(EMI[EMI!=0])) - - masked_FRP = np.ma.masked_array(FRP, FRP==0, fill_value=-1) - masked_SD = np.ma.masked_array(FRP_SD, FRP_SD==0, fill_value=-1) - masked_FRE = np.ma.masked_array(FRE, FRE==0, fill_value=-1) - masked_EMI = np.ma.masked_array(EMI, EMI==0, fill_value=-1) - - - ## write outputs - f = Dataset(f_output, 'w') - f.createDimension('time', 1) - f.createDimension('grid_yt', LAT.shape[0]) - f.createDimension('grid_xt', LAT.shape[1]) - - var_time = f.createVariable('time', 'i4', ('time',)) - var_lat = f.createVariable('grid_latt', 'f4', ('grid_yt', 'grid_xt')) - var_lon = f.createVariable('grid_lont', 'f4', ('grid_yt', 'grid_xt')) - var_frp = f.createVariable('FRP_MEAN', 'f4', ('time', 'grid_yt', 'grid_xt'), fill_value=-1) - var_sd = f.createVariable('FRP_SD', 'f4', ('time', 'grid_yt', 'grid_xt'), fill_value=-1) - var_fre = f.createVariable('FRE', 'f4', ('time', 'grid_yt', 'grid_xt'), fill_value=-1) - var_emi = f.createVariable('PM2.5', 'f4', ('time', 'grid_yt', 'grid_xt'), fill_value=-1) - - var_time[:] = f_time - var_lat[:] = np.flip(LAT) - var_lon[:] = LON - var_frp[:] = np.flipud(masked_FRP) - var_sd[:] = np.flipud(masked_SD) - var_fre[:] = np.flipud(masked_FRE) - var_emi[:] = np.flipud(masked_EMI) - f.close() - - print(i+1, '/', NN) From 1f78e3a282ffbaf91484881cdec36a76eb7412b7 Mon Sep 17 00:00:00 2001 From: Wei-Ting Hung <107704243+angehung5@users.noreply.github.com> Date: Tue, 3 Sep 2024 14:08:03 -0400 Subject: [PATCH 02/10] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5b5e412..7243765 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Repository for a fire spread forecast model, generating hourly fire radiative po ## Components -Trained machine learning models are available on Hera: `/scratch1/RDARCH/rda-arl-gpu/Wei-ting.Hung/fira_models.zip` +Trained machine learning models are available in `model/fira_models.zip` | **Source Code (Python Script)** | **Script Description** | | --------------------------------- | ------------------------------------------------------------------------------------------------------------------ | From 1286c6ae940e455a14850605516f48379513f3ca Mon Sep 17 00:00:00 2001 From: zmoon Date: Thu, 5 Sep 2024 16:36:17 -0500 Subject: [PATCH 03/10] datetime is part of Python --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 7243765..81de668 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,12 @@ Repository for a fire spread forecast model, generating hourly fire radiative power (FRP) prediction for air quality forecasting applications. -## Requred python libraries +## Required Python libraries -- Numpy +- NumPy - pandas - SciPy - MetPy -- datetime - netCDF4 - Matplotlib (for output plotting only) - Tensorflow From 6b411f6718e5111dd69daddd39947b8945861764 Mon Sep 17 00:00:00 2001 From: zmoon Date: Thu, 5 Sep 2024 16:37:22 -0500 Subject: [PATCH 04/10] xarray used in inputgen --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 81de668..78e3e59 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ Repository for a fire spread forecast model, generating hourly fire radiative po - NumPy - pandas +- xarray - SciPy - MetPy - netCDF4 From decaf053db37255ae8e641bb9a2204066efb28d1 Mon Sep 17 00:00:00 2001 From: zmoon Date: Thu, 5 Sep 2024 16:40:52 -0500 Subject: [PATCH 05/10] style --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 78e3e59..6ccfacc 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Trained machine learning models are available in `model/fira_models.zip` 1. Filename format: `[input name].[start time].f[simuluation time].nc` `input name` and `start time` can be specified in namelist. -2. Input netcdf file components: +2. Input netCDF file components: | **Variable Name** | **Variable Description** | | ------------------- | ---------------------------------------------------------------------------------- | @@ -110,7 +110,7 @@ Trained machine learning models are available in `model/fira_models.zip` 1. Filename format: `[output name].[start time].f[forecast hour].nc` `output name` and `start time` can be specified in namelist. -2. Output netcdf file components: +2. Output netCDF file components: | **Variable Name** | **Variable Description** | | ------------------- | ---------------------------------------------------------------------- | From c5050fd6598a488cd5ec5cefa024ea43edebe707 Mon Sep 17 00:00:00 2001 From: zmoon Date: Mon, 9 Sep 2024 15:04:39 -0500 Subject: [PATCH 06/10] Alphabetical --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 90f8530..1f046fb 100644 --- a/README.md +++ b/README.md @@ -4,15 +4,15 @@ Repository for a fire spread forecast model, generating hourly fire radiative po ## Required Python libraries +- Keras +- Matplotlib (for output plotting only) +- MetPy +- netCDF4 - NumPy - pandas -- xarray - SciPy -- MetPy -- netCDF4 -- Matplotlib (for output plotting only) - Tensorflow -- Keras +- xarray - XGBoost ## Components From 8c7ed593b82cd68caa3c8fdccb592f2b1c69c2e4 Mon Sep 17 00:00:00 2001 From: zmoon Date: Mon, 9 Sep 2024 15:07:45 -0500 Subject: [PATCH 07/10] Some flake8 formatting ignores --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9de84a7..3f308d0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,7 +28,7 @@ repos: rev: "7.1.1" hooks: - id: flake8 - args: [--max-line-length=88] + args: [--max-line-length=88, "--ignore=E203,W503"] - repo: https://github.com/codespell-project/codespell rev: v2.3.0 From c9f2bc6b442ad0f8bc58cbbf92c2a427298c2395 Mon Sep 17 00:00:00 2001 From: zmoon Date: Mon, 9 Sep 2024 15:14:50 -0500 Subject: [PATCH 08/10] Ignore undefined should get defined during the "fire frame" loop --- src/fire_inputgen.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/fire_inputgen.py b/src/fire_inputgen.py index c1a1578..fdb8bc3 100755 --- a/src/fire_inputgen.py +++ b/src/fire_inputgen.py @@ -681,9 +681,21 @@ def main_driver(initial_hour, forecast_hour, f_input, f_output, lat_lim, lon_lim ] if "INPUTFRAME" in locals(): - INPUTFRAME = np.append(INPUTFRAME, np.expand_dims(X_fire, axis=0), axis=0) - LATFRAME = np.append(LATFRAME, np.expand_dims(X_lat, axis=0), axis=0) - LONFRAME = np.append(LONFRAME, np.expand_dims(X_lon, axis=0), axis=0) + INPUTFRAME = np.append( + INPUTFRAME, + np.expand_dims(X_fire, axis=0), + axis=0, + ) # noqa: F821 + LATFRAME = np.append( + LATFRAME, + np.expand_dims(X_lat, axis=0), + axis=0, + ) # noqa: F821 + LONFRAME = np.append( + LONFRAME, + np.expand_dims(X_lon, axis=0), + axis=0, + ) # noqa: F821 else: INPUTFRAME = np.expand_dims(X_fire, axis=0) LATFRAME = np.expand_dims(X_lat, axis=0) From b82d8ee00471c07b7dcb8db984024f96d42e5f73 Mon Sep 17 00:00:00 2001 From: zmoon Date: Mon, 9 Sep 2024 15:18:06 -0500 Subject: [PATCH 09/10] Alternative approach since otherwise it complains about the vars not being defined later --- src/fire_inputgen.py | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/src/fire_inputgen.py b/src/fire_inputgen.py index fdb8bc3..dc900e5 100755 --- a/src/fire_inputgen.py +++ b/src/fire_inputgen.py @@ -645,6 +645,7 @@ def main_driver(initial_hour, forecast_hour, f_input, f_output, lat_lim, lon_lim lw[lw == 0] = np.nan # fire frame + INPUTFRAME = LATFRAME = LONFRAME = None for j in np.arange(1, num + 1, 1): total = total + 1 index = np.argwhere(lw == j) @@ -680,22 +681,10 @@ def main_driver(initial_hour, forecast_hour, f_input, f_output, lat_lim, lon_lim loc[0] - fsize : loc[0] + fsize + 1, loc[1] - fsize : loc[1] + fsize + 1 ] - if "INPUTFRAME" in locals(): - INPUTFRAME = np.append( - INPUTFRAME, - np.expand_dims(X_fire, axis=0), - axis=0, - ) # noqa: F821 - LATFRAME = np.append( - LATFRAME, - np.expand_dims(X_lat, axis=0), - axis=0, - ) # noqa: F821 - LONFRAME = np.append( - LONFRAME, - np.expand_dims(X_lon, axis=0), - axis=0, - ) # noqa: F821 + if INPUTFRAME is not None: + INPUTFRAME = np.append(INPUTFRAME, np.expand_dims(X_fire, axis=0), axis=0) + LATFRAME = np.append(LATFRAME, np.expand_dims(X_lat, axis=0), axis=0) + LONFRAME = np.append(LONFRAME, np.expand_dims(X_lon, axis=0), axis=0) else: INPUTFRAME = np.expand_dims(X_fire, axis=0) LATFRAME = np.expand_dims(X_lat, axis=0) From f571d6881657548350577adc77dea8bdec20166a Mon Sep 17 00:00:00 2001 From: zmoon Date: Mon, 9 Sep 2024 15:19:55 -0500 Subject: [PATCH 10/10] Ignore "readin" --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3f308d0..c9fc908 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,3 +34,4 @@ repos: rev: v2.3.0 hooks: - id: codespell + args: [--ignore-words-list=readin]