From 1a275c67acc6b58ef72632f77829798ade2482bf Mon Sep 17 00:00:00 2001 From: George McCabe Date: Wed, 24 Mar 2021 15:41:19 -0600 Subject: [PATCH 01/43] add support for climo_cdf dictionary in GridStat --- metplus/wrappers/compare_gridded_wrapper.py | 24 +++++++++++++++++++ metplus/wrappers/ensemble_stat_wrapper.py | 23 ------------------ metplus/wrappers/grid_stat_wrapper.py | 3 +++ parm/met_config/GridStatConfig_wrapped | 7 ++---- .../met_tool_wrapper/GridStat/GridStat.conf | 4 ++++ ...ridStat_fcstGFS_obsGFS_Sfc_MultiField.conf | 4 +++- ...t_fcstGFS_obsGFS_climoNCEP_MultiField.conf | 4 +++- .../GridStat_fcstGFS_obsCCPA_GRIB.conf | 4 +++- 8 files changed, 42 insertions(+), 31 deletions(-) diff --git a/metplus/wrappers/compare_gridded_wrapper.py b/metplus/wrappers/compare_gridded_wrapper.py index b0300216a7..5393964a96 100755 --- a/metplus/wrappers/compare_gridded_wrapper.py +++ b/metplus/wrappers/compare_gridded_wrapper.py @@ -405,3 +405,27 @@ def get_command(self): cmd += '-outdir {}'.format(self.outdir) return cmd + + def handle_climo_cdf_dict(self): + app_name_upper = self.app_name.upper() + tmp_dict = {} + self.set_met_config_float(tmp_dict, + f'{app_name_upper}_CLIMO_CDF_BINS', + 'cdf_bins', + 'CLIMO_CDF_BINS') + self.set_met_config_bool(tmp_dict, + f'{app_name_upper}_CLIMO_CDF_CENTER_BINS', + 'center_bins', + 'CLIMO_CDF_CENTER_BINS') + self.set_met_config_bool(tmp_dict, + f'{app_name_upper}_CLIMO_CDF_WRITE_BINS', + 'write_bins', + 'CLIMO_CDF_WRITE_BINS') + climo_cdf = ( + self.format_met_config_dict(tmp_dict, + 'climo_cdf', + ['CLIMO_CDF_BINS', + 'CLIMO_CDF_CENTER_BINS', + 'CLIMO_CDF_WRITE_BINS']) + ) + self.env_var_dict['METPLUS_CLIMO_CDF_DICT'] = climo_cdf diff --git a/metplus/wrappers/ensemble_stat_wrapper.py b/metplus/wrappers/ensemble_stat_wrapper.py index 485c46b879..69fa693f12 100755 --- a/metplus/wrappers/ensemble_stat_wrapper.py +++ b/metplus/wrappers/ensemble_stat_wrapper.py @@ -387,29 +387,6 @@ def handle_nbrhd_prob_dict(self): ) self.env_var_dict['METPLUS_NBRHD_PROB_DICT'] = nbrhd_prob - def handle_climo_cdf_dict(self): - tmp_dict = {} - self.set_met_config_float(tmp_dict, - 'ENSEMBLE_STAT_CLIMO_CDF_BINS', - 'cdf_bins', - 'CLIMO_CDF_BINS') - self.set_met_config_bool(tmp_dict, - 'ENSEMBLE_STAT_CLIMO_CDF_CENTER_BINS', - 'center_bins', - 'CLIMO_CDF_CENTER_BINS') - self.set_met_config_bool(tmp_dict, - 'ENSEMBLE_STAT_CLIMO_CDF_WRITE_BINS', - 'write_bins', - 'CLIMO_CDF_WRITE_BINS') - climo_cdf = ( - self.format_met_config_dict(tmp_dict, - 'climo_cdf', - ['CLIMO_CDF_BINS', - 'CLIMO_CDF_CENTER_BINS', - 'CLIMO_CDF_WRITE_BINS']) - ) - self.env_var_dict['METPLUS_CLIMO_CDF_DICT'] = climo_cdf - def handle_interp_dict(self): tmp_dict = {} self.set_met_config_string(tmp_dict, diff --git a/metplus/wrappers/grid_stat_wrapper.py b/metplus/wrappers/grid_stat_wrapper.py index f1354d11f8..7a6c6437a6 100755 --- a/metplus/wrappers/grid_stat_wrapper.py +++ b/metplus/wrappers/grid_stat_wrapper.py @@ -39,6 +39,7 @@ class GridStatWrapper(CompareGriddedWrapper): 'METPLUS_NBRHD_WIDTH', 'METPLUS_NBRHD_COV_THRESH', 'METPLUS_OUTPUT_PREFIX', + 'METPLUS_CLIMO_CDF_DICT', ] def __init__(self, config, instance=None, config_overrides={}): @@ -137,6 +138,8 @@ def create_c_dict(self): self.handle_mask(single_value=False, c_dict=c_dict) + self.handle_climo_cdf_dict() + return c_dict def set_environment_variables(self, time_info): diff --git a/parm/met_config/GridStatConfig_wrapped b/parm/met_config/GridStatConfig_wrapped index 45f9174891..b916d0f3b8 100644 --- a/parm/met_config/GridStatConfig_wrapped +++ b/parm/met_config/GridStatConfig_wrapped @@ -97,11 +97,8 @@ climo_stdev = { // // May be set separately in each "obs.field" entry // -climo_cdf = { - cdf_bins = 1; - center_bins = FALSE; - write_bins = TRUE; -} +//climo_cdf = { +${METPLUS_CLIMO_CDF_DICT} //////////////////////////////////////////////////////////////////////////////// diff --git a/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf b/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf index 0d19cc6f68..36ca42c399 100644 --- a/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf +++ b/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf @@ -132,6 +132,10 @@ OBS_GRID_STAT_PROB_THRESH = ==0.1 GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_vs_{OBTYPE}_{CURRENT_OBS_NAME} +#GRID_STAT_CLIMO_CDF_BINS = 1 +#GRID_STAT_CLIMO_CDF_CENTER_BINS = False +#GRID_STAT_CLIMO_CDF_WRITE_BINS = True + # End of [config] section and start of [dir] section [dir] diff --git a/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf b/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf index f2190d83a8..5340905f59 100644 --- a/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf +++ b/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf @@ -129,7 +129,9 @@ GRID_STAT_MASK_POLY = {INPUT_BASE}/model_applications/medium_range/poly/NHX.nc, {INPUT_BASE}/model_applications/medium_range/poly/CAM.nc, {INPUT_BASE}/model_applications/medium_range/poly/NSA.nc -GRID_STAT_MET_CONFIG_OVERRIDES = climo_mean = { regrid = { method = BILIN;width = 2;} day_interval = 1; time_interp_method = NEAREST;} output_flag = { ctc = NONE; cts = NONE; sl1l2 = STAT; eclv = NONE; grad = NONE;} grid_weight_flag = COS_LAT; climo_cdf = { write_bins=FALSE;} +GRID_STAT_CLIMO_CDF_WRITE_BINS = False + +GRID_STAT_MET_CONFIG_OVERRIDES = climo_mean = { regrid = { method = BILIN;width = 2;} day_interval = 1; time_interp_method = NEAREST;} output_flag = { ctc = NONE; cts = NONE; sl1l2 = STAT; eclv = NONE; grad = NONE;} grid_weight_flag = COS_LAT; # variables to describe format of forecast data FCST_IS_PROB = false diff --git a/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf b/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf index ebb6ff6091..bfb90808d0 100644 --- a/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf +++ b/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf @@ -59,7 +59,9 @@ GRID_STAT_REGRID_WIDTH = 2 GRID_STAT_MASK_GRID = FULL GRID_STAT_MASK_POLY = {INPUT_BASE}/model_applications/medium_range/poly/NHX.nc, {INPUT_BASE}/model_applications/medium_range/poly/SHX.nc, {INPUT_BASE}/model_applications/medium_range/poly/TRO.nc, {INPUT_BASE}/model_applications/medium_range/poly/PNA.nc -GRID_STAT_MET_CONFIG_OVERRIDES = climo_mean = { regrid = { method = BILIN;width = 2;} day_interval = 1; time_interp_method = NEAREST;} climo_cdf = { write_bins=FALSE;} output_flag = { ctc = NONE; cts = NONE; sal1l2 = STAT; val1l2 = STAT; eclv = NONE; grad = NONE;} grid_weight_flag = COS_LAT; climo_mean = fcst; +GRID_STAT_CLIMO_CDF_WRITE_BINS = False + +GRID_STAT_MET_CONFIG_OVERRIDES = climo_mean = { regrid = { method = BILIN;width = 2;} day_interval = 1; time_interp_method = NEAREST;} output_flag = { ctc = NONE; cts = NONE; sal1l2 = STAT; val1l2 = STAT; eclv = NONE; grad = NONE;} grid_weight_flag = COS_LAT; climo_mean = fcst; # variables to describe format of forecast data FCST_IS_PROB = false diff --git a/parm/use_cases/model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.conf b/parm/use_cases/model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.conf index af0a4dc5b7..56782ad403 100644 --- a/parm/use_cases/model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.conf +++ b/parm/use_cases/model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.conf @@ -56,7 +56,9 @@ GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_vs_{OBTYPE}_{CURRENT_OBS_N GRID_STAT_MASK_POLY = {INPUT_BASE}/model_applications/precipitation/poly/CONUS.nc, {INPUT_BASE}/model_applications/precipitation/poly/EAST.nc, {INPUT_BASE}/model_applications/precipitation/poly/WEST.nc -GRID_STAT_MET_CONFIG_OVERRIDES = climo_mean = { regrid = { method = BILIN;width = 2;} time_interp_method = NEAREST;} climo_cdf = { write_bins=FALSE;} output_flag = { cts = NONE; eclv = NONE; grad = NONE;} +GRID_STAT_CLIMO_CDF_WRITE_BINS = False + +GRID_STAT_MET_CONFIG_OVERRIDES = climo_mean = { regrid = { method = BILIN;width = 2;} time_interp_method = NEAREST;} output_flag = { cts = NONE; eclv = NONE; grad = NONE;} # Forecast data description variables FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB From c8a2731fae3527e7d08a0c428fa1e9dff2aa5bff Mon Sep 17 00:00:00 2001 From: George McCabe Date: Wed, 24 Mar 2021 15:45:29 -0600 Subject: [PATCH 02/43] add support for climo_cdf dictionary in PointStat, ci-run-diff --- metplus/wrappers/point_stat_wrapper.py | 3 +++ parm/met_config/PointStatConfig_wrapped | 7 ++----- parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf | 4 ++++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/metplus/wrappers/point_stat_wrapper.py b/metplus/wrappers/point_stat_wrapper.py index 80a629b371..5e8ab2f347 100755 --- a/metplus/wrappers/point_stat_wrapper.py +++ b/metplus/wrappers/point_stat_wrapper.py @@ -34,6 +34,7 @@ class PointStatWrapper(CompareGriddedWrapper): 'METPLUS_MASK_POLY', 'METPLUS_MASK_SID', 'METPLUS_OUTPUT_PREFIX', + 'METPLUS_CLIMO_CDF_DICT', ] def __init__(self, config, instance=None, config_overrides={}): @@ -137,6 +138,8 @@ def create_c_dict(self): 'message_type', 'METPLUS_MESSAGE_TYPE',) + self.handle_climo_cdf_dict() + c_dict['OBS_VALID_BEG'] = ( self.config.getraw('config', 'POINT_STAT_OBS_VALID_BEG', '') ) diff --git a/parm/met_config/PointStatConfig_wrapped b/parm/met_config/PointStatConfig_wrapped index 2042f7b982..a1b943f0e7 100644 --- a/parm/met_config/PointStatConfig_wrapped +++ b/parm/met_config/PointStatConfig_wrapped @@ -102,11 +102,8 @@ ${METPLUS_CLIMO_STDEV_FILE} // // May be set separately in each "obs.field" entry // -climo_cdf = { - cdf_bins = 1; - center_bins = FALSE; - write_bins = TRUE; -} +//climo_cdf = { +${METPLUS_CLIMO_CDF_DICT} //////////////////////////////////////////////////////////////////////////////// diff --git a/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf b/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf index 25b193aee1..fe60e962b6 100644 --- a/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf +++ b/parm/use_cases/met_tool_wrapper/PointStat/PointStat.conf @@ -50,6 +50,10 @@ LOOP_ORDER = processes # or the value of the environment variable METPLUS_PARM_BASE if set POINT_STAT_CONFIG_FILE ={PARM_BASE}/met_config/PointStatConfig_wrapped +#POINT_STAT_CLIMO_CDF_BINS = 1 +#POINT_STAT_CLIMO_CDF_CENTER_BINS = False +#POINT_STAT_CLIMO_CDF_WRITE_BINS = True + # Time relative to each input file's valid time (in seconds if no units are specified) for data within the file to be # considered valid. Values are set in the 'obs_window' dictionary in the PointStat config file OBS_POINT_STAT_WINDOW_BEGIN = -5400 From a43d4add7189f192d6bc82694dc2bebf9110610f Mon Sep 17 00:00:00 2001 From: George McCabe Date: Wed, 24 Mar 2021 17:22:45 -0600 Subject: [PATCH 03/43] added support for output_flag and nc_pairs_flag in GridStat - updated some configs to reflect chnage --- metplus/wrappers/grid_stat_wrapper.py | 41 +++++++++++++++++++ .../met_tool_wrapper/GridStat/GridStat.conf | 34 +++++++++++++++ .../GridStat_fcstCESM_obsGFS_ConusTemp.conf | 7 +++- ...at_fcstFV3_obsGOES_BrightnessTempDmap.conf | 12 +++++- ...stHRRR_obsPracPerfect_SurrogateSevere.conf | 5 ++- ...RR_obsPracPerfect_SurrogateSevereProb.conf | 10 ++++- ...ridStat_fcstGFS_obsGFS_Sfc_MultiField.conf | 8 +++- ...t_fcstGFS_obsGFS_climoNCEP_MultiField.conf | 9 +++- .../GridStat_fcstGFS_obsCCPA_GRIB.conf | 6 ++- ...sis_fcstNMME_obsCPC_seasonal_forecast.conf | 13 +++++- 10 files changed, 137 insertions(+), 8 deletions(-) diff --git a/metplus/wrappers/grid_stat_wrapper.py b/metplus/wrappers/grid_stat_wrapper.py index 7a6c6437a6..ff6b64bac9 100755 --- a/metplus/wrappers/grid_stat_wrapper.py +++ b/metplus/wrappers/grid_stat_wrapper.py @@ -40,8 +40,46 @@ class GridStatWrapper(CompareGriddedWrapper): 'METPLUS_NBRHD_COV_THRESH', 'METPLUS_OUTPUT_PREFIX', 'METPLUS_CLIMO_CDF_DICT', + 'METPLUS_OUTPUT_FLAG_DICT', + 'METPLUS_NC_PAIRS_FLAG_DICT', ] + OUTPUT_FLAGS = ['fho', + 'ctc', + 'cts', + 'mctc', + 'mcts', + 'cnt', + 'sl1l2', + 'sal1l2', + 'vl1l2', + 'val1l2', + 'vcnt', + 'pct', + 'pstd', + 'pjc', + 'prc', + 'eclv', + 'nbrctc', + 'nbrcts', + 'nbrcnt', + 'grad', + 'dmap', + ] + + NC_PAIRS_FLAGS = ['latlon', + 'raw', + 'diff', + 'climo', + 'climo_cdp', + 'weight', + 'nbrhd', + 'fourier', + 'gradient', + 'distance_map', + 'apply_mask', + ] + def __init__(self, config, instance=None, config_overrides={}): self.app_name = 'grid_stat' self.app_path = os.path.join(config.getdir('MET_BIN_DIR', ''), @@ -140,6 +178,9 @@ def create_c_dict(self): self.handle_climo_cdf_dict() + self.handle_flags('output') + self.handle_flags('nc_pairs') + return c_dict def set_environment_variables(self, time_info): diff --git a/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf b/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf index 36ca42c399..61154d8700 100644 --- a/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf +++ b/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf @@ -136,6 +136,40 @@ GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_vs_{OBTYPE}_{CURRENT_OBS_N #GRID_STAT_CLIMO_CDF_CENTER_BINS = False #GRID_STAT_CLIMO_CDF_WRITE_BINS = True +#GRID_STAT_OUTPUT_FLAG_FHO = NONE +#GRID_STAT_OUTPUT_FLAG_CTC = STAT +#GRID_STAT_OUTPUT_FLAG_CTS = STAT +#GRID_STAT_OUTPUT_FLAG_MCTC = NONE +#GRID_STAT_OUTPUT_FLAG_MCTS = NONE +#GRID_STAT_OUTPUT_FLAG_CNT = NONE +#GRID_STAT_OUTPUT_FLAG_SL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_SAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VAL1L2 = NONE +#GRID_STAT_OUTPUT_FLAG_VCNT = NONE +#GRID_STAT_OUTPUT_FLAG_PCT = NONE +#GRID_STAT_OUTPUT_FLAG_PSTD = NONE +#GRID_STAT_OUTPUT_FLAG_PJC = NONE +#GRID_STAT_OUTPUT_FLAG_PRC = NONE +#GRID_STAT_OUTPUT_FLAG_ECLV = BOTH +#GRID_STAT_OUTPUT_FLAG_NBRCTC = NONE +#GRID_STAT_OUTPUT_FLAG_NBRCTS = NONE +#GRID_STAT_OUTPUT_FLAG_NBRCNT = NONE +#GRID_STAT_OUTPUT_FLAG_GRAD = BOTH +#GRID_STAT_OUTPUT_FLAG_DMAP = NONE + +#GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +#GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +#GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +#GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +#GRID_STAT_NC_PAIRS_FLAG_CLIMO_CDP = FALSE +#GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE +#GRID_STAT_NC_PAIRS_FLAG_NBRHD = FALSE +#GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE +#GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE +#GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE +#GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE + # End of [config] section and start of [dir] section [dir] diff --git a/parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf b/parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf index 60731e7dda..dc2d27fc97 100644 --- a/parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf +++ b/parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf @@ -80,7 +80,12 @@ OBS_IS_PROB = false # Output prefix set in grid_stat config file GRID_STAT_OUTPUT_PREFIX={MODEL}_{CURRENT_OBS_NAME}_vs_{OBTYPE} -GRID_STAT_MET_CONFIG_OVERRIDES = interp = { field = NONE;} output_flag = {cnt = STAT; sl1l2 = STAT;eclv = NONE;grad = NONE; } +GRID_STAT_OUTPUT_FLAG_CNT = STAT +GRID_STAT_OUTPUT_FLAG_SL1L2 = STAT +GRID_STAT_OUTPUT_FLAG_ECLV = NONE +GRID_STAT_OUTPUT_FLAG_GRAD = NONE + +GRID_STAT_MET_CONFIG_OVERRIDES = interp = { field = NONE;} # End of [config] section and start of [dir] section [dir] diff --git a/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf b/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf index b7c0b936e0..9e3c4bca65 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf @@ -40,7 +40,17 @@ GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped GRID_STAT_OUTPUT_PREFIX = FV3_core_{instance} -GRID_STAT_MET_CONFIG_OVERRIDES = output_flag = {ctc = NONE;cts = NONE; eclv = NONE;grad = NONE;dmap = BOTH;} nc_pairs_flag = {latlon = TRUE;raw = TRUE;diff = TRUE; distance_map = TRUE; apply_mask = TRUE;} +GRID_STAT_OUTPUT_FLAG_CTC = NONE +GRID_STAT_OUTPUT_FLAG_CTS = NONE +GRID_STAT_OUTPUT_FLAG_ECLV = NONE +GRID_STAT_OUTPUT_FLAG_GRAD = NONE +GRID_STAT_OUTPUT_FLAG_DMAP = DMAP + +GRID_STAT_NC_PAIRS_FLAG_LATLON = TRUE +GRID_STAT_NC_PAIRS_FLAG_RAW = TRUE +GRID_STAT_NC_PAIRS_FLAG_DIFF = TRUE +GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = TRUE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = TRUE [dir] # Input and Output Diretory of the object data diff --git a/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.conf b/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.conf index a9b98a8377..12bb5264a5 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.conf @@ -80,7 +80,10 @@ FCST_IS_PROB = false GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped -GRID_STAT_MET_CONFIG_OVERRIDES = output_flag = {ctc = BOTH; cts = BOTH; eclv = NONE; grad = NONE; } +GRID_STAT_OUTPUT_FLAG_CTC = BOTH +GRID_STAT_OUTPUT_FLAG_CTS = BOTH +GRID_STAT_OUTPUT_FLAG_ECLV = NONE +GRID_STAT_OUTPUT_FLAG_GRAD = NONE [dir] diff --git a/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.conf b/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.conf index 8337c3952b..80d4002ccc 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.conf @@ -83,7 +83,15 @@ FCST_GRID_STAT_INPUT_DATATYPE = NETCDF GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped -GRID_STAT_MET_CONFIG_OVERRIDES = output_flag = {ctc = NONE; cts = NONE; pct = BOTH; pstd = BOTH; pjc = BOTH; prc = BOTH; eclv = NONE; grad = NONE; } +GRID_STAT_OUTPUT_FLAG_CTC = NONE +GRID_STAT_OUTPUT_FLAG_CTS = NONE +GRID_STAT_OUTPUT_FLAG_PCT = BOTH +GRID_STAT_OUTPUT_FLAG_PSTD = BOTH +GRID_STAT_OUTPUT_FLAG_PJC = BOTH +GRID_STAT_OUTPUT_FLAG_PRC = BOTH +GRID_STAT_OUTPUT_FLAG_ECLV = NONE +GRID_STAT_OUTPUT_FLAG_GRAD = NONE + [dir] # input and output data directories for each application in PROCESS_LIST diff --git a/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf b/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf index 5340905f59..ec97313002 100644 --- a/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf +++ b/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf @@ -131,7 +131,13 @@ GRID_STAT_MASK_POLY = {INPUT_BASE}/model_applications/medium_range/poly/NHX.nc, GRID_STAT_CLIMO_CDF_WRITE_BINS = False -GRID_STAT_MET_CONFIG_OVERRIDES = climo_mean = { regrid = { method = BILIN;width = 2;} day_interval = 1; time_interp_method = NEAREST;} output_flag = { ctc = NONE; cts = NONE; sl1l2 = STAT; eclv = NONE; grad = NONE;} grid_weight_flag = COS_LAT; +GRID_STAT_OUTPUT_FLAG_CTC = NONE +GRID_STAT_OUTPUT_FLAG_CTS = NONE +GRID_STAT_OUTPUT_FLAG_SL1L2 = STAT +GRID_STAT_OUTPUT_FLAG_ECLV = NONE +GRID_STAT_OUTPUT_FLAG_GRAD = NONE + +GRID_STAT_MET_CONFIG_OVERRIDES = climo_mean = { regrid = { method = BILIN;width = 2;} day_interval = 1; time_interp_method = NEAREST;} grid_weight_flag = COS_LAT; # variables to describe format of forecast data FCST_IS_PROB = false diff --git a/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf b/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf index bfb90808d0..3d8efe55c4 100644 --- a/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf +++ b/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf @@ -61,7 +61,14 @@ GRID_STAT_MASK_POLY = {INPUT_BASE}/model_applications/medium_range/poly/NHX.nc, GRID_STAT_CLIMO_CDF_WRITE_BINS = False -GRID_STAT_MET_CONFIG_OVERRIDES = climo_mean = { regrid = { method = BILIN;width = 2;} day_interval = 1; time_interp_method = NEAREST;} output_flag = { ctc = NONE; cts = NONE; sal1l2 = STAT; val1l2 = STAT; eclv = NONE; grad = NONE;} grid_weight_flag = COS_LAT; climo_mean = fcst; +GRID_STAT_OUTPUT_FLAG_CTC = NONE +GRID_STAT_OUTPUT_FLAG_CTS = NONE +GRID_STAT_OUTPUT_FLAG_SAL1L2 = STAT +GRID_STAT_OUTPUT_FLAG_VAL1L2 = STAT +GRID_STAT_OUTPUT_FLAG_ECLV = NONE +GRID_STAT_OUTPUT_FLAG_GRAD = NONE + +GRID_STAT_MET_CONFIG_OVERRIDES = climo_mean = { regrid = { method = BILIN;width = 2;} day_interval = 1; time_interp_method = NEAREST;} grid_weight_flag = COS_LAT; climo_mean = fcst; # variables to describe format of forecast data FCST_IS_PROB = false diff --git a/parm/use_cases/model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.conf b/parm/use_cases/model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.conf index 56782ad403..3641e9edff 100644 --- a/parm/use_cases/model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.conf +++ b/parm/use_cases/model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.conf @@ -58,7 +58,11 @@ GRID_STAT_MASK_POLY = {INPUT_BASE}/model_applications/precipitation/poly/CONUS.n GRID_STAT_CLIMO_CDF_WRITE_BINS = False -GRID_STAT_MET_CONFIG_OVERRIDES = climo_mean = { regrid = { method = BILIN;width = 2;} time_interp_method = NEAREST;} output_flag = { cts = NONE; eclv = NONE; grad = NONE;} +GRID_STAT_OUTPUT_FLAG_CTS = NONE +GRID_STAT_OUTPUT_FLAG_ECLV = NONE +GRID_STAT_OUTPUT_FLAG_GRAD = NONE + +GRID_STAT_MET_CONFIG_OVERRIDES = climo_mean = { regrid = { method = BILIN;width = 2;} time_interp_method = NEAREST;} # Forecast data description variables FCST_PCP_COMBINE_INPUT_DATATYPE = GRIB diff --git a/parm/use_cases/model_applications/s2s/GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.conf b/parm/use_cases/model_applications/s2s/GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.conf index 418a1f9b80..3672a5b56c 100644 --- a/parm/use_cases/model_applications/s2s/GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.conf +++ b/parm/use_cases/model_applications/s2s/GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.conf @@ -53,8 +53,19 @@ OBTYPE = CPC # location of grid_stat MET config file GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped +GRID_STAT_OUTPUT_FLAG_CTS = NONE +GRID_STAT_OUTPUT_FLAG_CNT = STAT +GRID_STAT_OUTPUT_FLAG_SL1L2 = STAT +GRID_STAT_OUTPUT_FLAG_ECLV = NONE +GRID_STAT_OUTPUT_FLAG_GRAD = NONE + +GRID_STAT_NC_PAIRS_FLAG_LATLON = TRUE +GRID_STAT_NC_PAIRS_FLAG_RAW = TRUE +GRID_STAT_NC_PAIRS_FLAG_DIFF = TRUE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = TRUE + # Override MET config file settings -GRID_STAT_MET_CONFIG_OVERRIDES = nc_pairs_flag = {latlon=TRUE; raw=TRUE; diff=TRUE; climo=TRUE;} output_flag = {cts=NONE; cnt=STAT; sl1l2=STAT; eclv=NONE; grad=NONE;} nc_pairs_var_name = "precip"; +GRID_STAT_MET_CONFIG_OVERRIDES = nc_pairs_var_name = "precip"; # variables to describe format of forecast data FCST_IS_PROB = false From c2675a7f0998115b9ec5a9ad725365d1cb0aba4d Mon Sep 17 00:00:00 2001 From: George McCabe Date: Thu, 25 Mar 2021 10:37:01 -0600 Subject: [PATCH 04/43] added more documentation about required arguments for function --- metplus/wrappers/command_builder.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/metplus/wrappers/command_builder.py b/metplus/wrappers/command_builder.py index 0297121174..ee5348e10d 100755 --- a/metplus/wrappers/command_builder.py +++ b/metplus/wrappers/command_builder.py @@ -2064,7 +2064,13 @@ def handle_met_config_dict(self, dict_name, dict_items, def add_met_config(self, **kwargs): """! Create METConfigInfo object from arguments and process @param kwargs key arguments that should match METConfigInfo - arguments + arguments, which includes the following: + @param name MET config variable name to set + @param data_type type of variable to set, i.e. string, list, bool + @param metplus_configs variables from METplus config that should + be read to get the value. This can be a list of variable names + in order of precedence (first variable is used if it is set, + otherwise 2nd variable is used if set, etc.) """ item = met_config(**kwargs) self.handle_met_config_item(item) From 07318627df2c7446b16c42e1d01c7da785d9548f Mon Sep 17 00:00:00 2001 From: George McCabe Date: Thu, 25 Mar 2021 10:37:25 -0600 Subject: [PATCH 05/43] add support for another METplus config name for climo_cdf.cdf_bins --- metplus/wrappers/compare_gridded_wrapper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metplus/wrappers/compare_gridded_wrapper.py b/metplus/wrappers/compare_gridded_wrapper.py index 5393964a96..aabaa0938a 100755 --- a/metplus/wrappers/compare_gridded_wrapper.py +++ b/metplus/wrappers/compare_gridded_wrapper.py @@ -410,7 +410,8 @@ def handle_climo_cdf_dict(self): app_name_upper = self.app_name.upper() tmp_dict = {} self.set_met_config_float(tmp_dict, - f'{app_name_upper}_CLIMO_CDF_BINS', + [f'{app_name_upper}_CLIMO_CDF_BINS', + f'{app_name_upper}_CLIMO_CDF_CDF_BINS'], 'cdf_bins', 'CLIMO_CDF_BINS') self.set_met_config_bool(tmp_dict, From 764ced3e2ed56a13e265d9b3882a21f4331363f3 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Thu, 25 Mar 2021 11:07:42 -0600 Subject: [PATCH 06/43] added utility to easy generate text that is needed to be added to documentation for adding support for a new MET config variable. moved get_wrapper_name logic into doc_util so it can be easily referenced by met_util but prevents a user from needing all of the requirements of met_util to run the doc utility --- metplus/util/__init__.py | 1 + metplus/util/doc_util.py | 145 +++++++++++++++++++++++++++++++++++++++ metplus/util/met_util.py | 51 +------------- 3 files changed, 147 insertions(+), 50 deletions(-) create mode 100755 metplus/util/doc_util.py diff --git a/metplus/util/__init__.py b/metplus/util/__init__.py index f5c2b7edf6..570ee45208 100644 --- a/metplus/util/__init__.py +++ b/metplus/util/__init__.py @@ -1,4 +1,5 @@ from .metplus_check import * +from .doc_util import * from .config_metplus import * from .time_util import * from .met_util import * diff --git a/metplus/util/doc_util.py b/metplus/util/doc_util.py new file mode 100755 index 0000000000..40be47e995 --- /dev/null +++ b/metplus/util/doc_util.py @@ -0,0 +1,145 @@ +#! /usr/bin/env python3 + +import sys +import os + +# dictionary used by get_wrapper_name function to easily convert wrapper +# name in many formats to the correct name of the wrapper class +LOWER_TO_WRAPPER_NAME = {'ascii2nc': 'ASCII2NC', + 'cycloneplotter': 'CyclonePlotter', + 'ensemblestat': 'EnsembleStat', + 'example': 'Example', + 'extracttiles': 'ExtractTiles', + 'gempaktocf': 'GempakToCF', + 'genvxmask': 'GenVxMask', + 'griddiag': 'GridDiag', + 'gridstat': 'GridStat', + 'makeplots': 'MakePlots', + 'mode': 'MODE', + 'mtd': 'MTD', + 'modetimedomain': 'MTD', + 'pb2nc': 'PB2NC', + 'pcpcombine': 'PCPCombine', + 'plotdataplane': 'PlotDataPlane', + 'point2grid': 'Point2Grid', + 'pointtogrid': 'Point2Grid', + 'pointstat': 'PointStat', + 'pyembedingest': 'PyEmbedIngest', + 'regriddataplane': 'RegridDataPlane', + 'seriesanalysis': 'SeriesAnalysis', + 'statanalysis': 'StatAnalysis', + 'tcgen': 'TCGen', + 'tcpairs': 'TCPairs', + 'tcrmw': 'TCRMW', + 'tcstat': 'TCStat', + 'tcmprplotter': 'TCMPRPlotter', + 'usage': 'Usage', + 'userscript': 'UserScript', + } + +def get_wrapper_name(process_name): + """! Determine name of wrapper from string that may not contain the correct + capitalization, i.e. Pcp-Combine translates to PCPCombine + + @param process_name string that was listed in the PROCESS_LIST + @returns name of wrapper (without 'Wrapper' at the end) and None if + name cannot be determined + """ + lower_process = (process_name.replace('-', '') + .replace('_', '') + .replace(' ', '') + .lower()) + if lower_process in LOWER_TO_WRAPPER_NAME.keys(): + return LOWER_TO_WRAPPER_NAME[lower_process] + + return None + + +def print_doc_text(tool_name, met_var, dict_items): + """! Format documentation for adding support for a new MET config variable + through METplus wrappers. + + @param tool_name MET tool name, i.e. grid_stat + @param met_var MET variable name, i.e. output_flag + @param dict_items (optional) list of MET dictionary var items if met_var + is a dictionary + """ + wrapper_caps = tool_name.upper() + met_var_caps = met_var.upper() + + wrapper_camel = get_wrapper_name(wrapper_caps) + + + metplus_var = f'{wrapper_caps}_{met_var_caps}' + + metplus_config_names = [] + met_config_values = [] + if not dict_items: + metplus_config_names.append(metplus_var) + met_config_values.append(met_var) + else: + for item_name in dict_items: + item_name_caps = item_name.upper() + metplus_config_name = f'{metplus_var}_{item_name_caps}' + + metplus_config_names.append(metplus_config_name) + met_config_values.append(f"{met_var}.{item_name}") + + print(f"Wrapper: {wrapper_camel}") + print(f"MET Variable: {met_var}") + if dict_items: + print(f"Dictionary Items:") + for item in dict_items: + print(f' {item}') + + print(f"\n\nIn docs/Users_Guide/wrappers.rst under {wrapper_camel} => " + "METplus Configuration section, add:\n\n") + for metplus_config_name in metplus_config_names: + print(f'| :term:`{metplus_config_name}`') + + print(f"\n\nIn docs/Users_Guide/wrappers.rst under {wrapper_camel} => " + "MET Configuration section, add:\n\n") + list_table_text = (".. list-table::\n" + " :widths: 5 5\n" + " :header-rows: 0\n\n" + " * - METplus Config(s)\n" + " - MET Config File\n" + ) + + for metplus_config_name, met_config_name in zip(metplus_config_names, met_config_values): + list_table_text += (f" * - :term:`{metplus_config_name}`\n" + f" - {met_config_name}\n" + ) + print(list_table_text) + + print(f"In docs/Users_Guide/glossary.rst, add:\n\n") + for metplus_config_name, met_config_name in zip(metplus_config_names, met_config_values): + glossary_entry = (f" {metplus_config_name}\n" + f" Specify the value for '{met_config_name}' " + f"in the MET configuration file for {wrapper_camel}.\n\n" + f" | *Used by:* {wrapper_camel}") + print(f'{glossary_entry}\n') + +def doc_util_usage(): + """! Print usage statement for script """ + print(f"{__file__} [ 3: + items = ','.join(sys.argv[3:]).split(',') + dict_items = [item.strip() for item in items] + + print_doc_text(tool_name, met_var, dict_items) diff --git a/metplus/util/met_util.py b/metplus/util/met_util.py index 5cb8e2a429..bab559b872 100644 --- a/metplus/util/met_util.py +++ b/metplus/util/met_util.py @@ -27,6 +27,7 @@ from .string_template_substitution import parse_template from .string_template_substitution import get_tags from . import time_util as time_util +from .doc_util import get_wrapper_name from .. import get_metplus_version @@ -39,39 +40,6 @@ PYTHON_EMBEDDING_TYPES = ['PYTHON_NUMPY', 'PYTHON_XARRAY', 'PYTHON_PANDAS'] -LOWER_TO_WRAPPER_NAME = {'ascii2nc': 'ASCII2NC', - 'cycloneplotter': 'CyclonePlotter', - 'ensemblestat': 'EnsembleStat', - 'example': 'Example', - 'extracttiles': 'ExtractTiles', - 'gempaktocf': 'GempakToCF', - 'genvxmask': 'GenVxMask', - 'griddiag': 'GridDiag', - 'gridstat': 'GridStat', - 'makeplots': 'MakePlots', - 'mode': 'MODE', - 'mtd': 'MTD', - 'modetimedomain': 'MTD', - 'pb2nc': 'PB2NC', - 'pcpcombine': 'PCPCombine', - 'plotdataplane': 'PlotDataPlane', - 'point2grid': 'Point2Grid', - 'pointtogrid': 'Point2Grid', - 'Point_2_Grid': 'Point2Grid', - 'pointstat': 'PointStat', - 'pyembedingest': 'PyEmbedIngest', - 'regriddataplane': 'RegridDataPlane', - 'seriesanalysis': 'SeriesAnalysis', - 'statanalysis': 'StatAnalysis', - 'tcgen': 'TCGen', - 'tcpairs': 'TCPairs', - 'tcrmw': 'TCRMW', - 'tcstat': 'TCStat', - 'tcmprplotter': 'TCMPRPlotter', - 'usage': 'Usage', - 'userscript': 'UserScript', - } - valid_comparisons = {">=": "ge", ">": "gt", "==": "eq", @@ -1780,23 +1748,6 @@ def get_process_list(config): return out_process_list -def get_wrapper_name(process_name): - """! Determine name of wrapper from string that may not contain the correct - capitalization, i.e. Pcp-Combine translates to PCPCombine - - @param process_name string that was listed in the PROCESS_LIST - @returns name of wrapper (without 'Wrapper' at the end) and None if - name cannot be determined - """ - lower_process = (process_name.replace('-', '') - .replace('_', '') - .replace(' ', '') - .lower()) - if lower_process in LOWER_TO_WRAPPER_NAME.keys(): - return LOWER_TO_WRAPPER_NAME[lower_process] - - return None - # minutes def shift_time(time_str, shift): """ Adjust time by shift hours. Format is %Y%m%d%H%M%S From e1fb77071f9a4afa9f1f8a3f1219ea2a606c6d44 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Thu, 25 Mar 2021 11:24:14 -0600 Subject: [PATCH 07/43] added missing header for MET configuration section documentation --- metplus/util/doc_util.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/metplus/util/doc_util.py b/metplus/util/doc_util.py index 40be47e995..db242a99c2 100755 --- a/metplus/util/doc_util.py +++ b/metplus/util/doc_util.py @@ -85,7 +85,7 @@ def print_doc_text(tool_name, met_var, dict_items): metplus_config_names.append(metplus_config_name) met_config_values.append(f"{met_var}.{item_name}") - print(f"Wrapper: {wrapper_camel}") + print(f"\nWrapper: {wrapper_camel}") print(f"MET Variable: {met_var}") if dict_items: print(f"Dictionary Items:") @@ -99,7 +99,12 @@ def print_doc_text(tool_name, met_var, dict_items): print(f"\n\nIn docs/Users_Guide/wrappers.rst under {wrapper_camel} => " "MET Configuration section, add:\n\n") - list_table_text = (".. list-table::\n" + var_header = (f"**${{METPLUS_{met_var_caps}" + f"{'_DICT' if dict_items else ''}" + "}**") + + list_table_text = (f"{var_header}\n\n" + ".. list-table::\n" " :widths: 5 5\n" " :header-rows: 0\n\n" " * - METplus Config(s)\n" From b1fbd6330ee1edabcfbc265a8182f5f7378fbaca Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 10:20:56 -0600 Subject: [PATCH 08/43] added other advice for adding support for new MET config variable --- metplus/util/doc_util.py | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/metplus/util/doc_util.py b/metplus/util/doc_util.py index db242a99c2..11e36f48e5 100755 --- a/metplus/util/doc_util.py +++ b/metplus/util/doc_util.py @@ -66,6 +66,7 @@ def print_doc_text(tool_name, met_var, dict_items): """ wrapper_caps = tool_name.upper() met_var_caps = met_var.upper() + env_var_name = f'METPLUS_{met_var_caps}' wrapper_camel = get_wrapper_name(wrapper_caps) @@ -78,6 +79,7 @@ def print_doc_text(tool_name, met_var, dict_items): metplus_config_names.append(metplus_var) met_config_values.append(met_var) else: + env_var_name = f'{env_var_name}_DICT' for item_name in dict_items: item_name_caps = item_name.upper() metplus_config_name = f'{metplus_var}_{item_name_caps}' @@ -85,6 +87,11 @@ def print_doc_text(tool_name, met_var, dict_items): metplus_config_names.append(metplus_config_name) met_config_values.append(f"{met_var}.{item_name}") + print('WARNING: Guidance output from this script may differ slightly ' + 'from the actual steps to take. It is intended to assist the process.' + ' The text that is generated should be reviewed for accuracy before ' + 'adding to codebase.') + print(f"\nWrapper: {wrapper_camel}") print(f"MET Variable: {met_var}") if dict_items: @@ -92,16 +99,39 @@ def print_doc_text(tool_name, met_var, dict_items): for item in dict_items: print(f' {item}') + print(f'\n\nIn the {tool_name}_wrapper.py file, in the {wrapper_camel}Wrapper ' + f'class, add the following to the WRAPPER_ENV_VAR_KEYS class ' + f"variable list:\n\n\n '{env_var_name}',\n\n") + + print(f'In the create_c_dict function for {wrapper_camel}Wrapper, add a ' + 'function call to read the new METplus config variables and save ' + 'the value to be added to the wrapped MET config file.\n\n') + if not dict_items: + print(f" self.add_met_config(name='{met_var}',\n" + " data_type='',\n" + f" metplus_configs=['{metplus_var}'])" + "\n\n" + "where can be string, list, int, float, bool, " + "or thresh.\n\n") + else: + print("Typically a function is written to handle MET config dictionary" + " items. Search for functions that start with handle_ in " + "CommandBuilder or other parent class wrappers to see if a " + "function already exists for the item you are adding or to use " + "as an example to write a new one.\n\n") + + print(f"In the parm/met_config/{wrapper_camel}Config_wrapped file, " + "replace:\n\n") + print(f"{met_var} = ...\n\n with:\n\n//{met_var} =\n${{{env_var_name}}}\n\n") + print(f"\n\nIn docs/Users_Guide/wrappers.rst under {wrapper_camel} => " - "METplus Configuration section, add:\n\n") + "METplus Configuration section, add:\n\n") for metplus_config_name in metplus_config_names: print(f'| :term:`{metplus_config_name}`') print(f"\n\nIn docs/Users_Guide/wrappers.rst under {wrapper_camel} => " "MET Configuration section, add:\n\n") - var_header = (f"**${{METPLUS_{met_var_caps}" - f"{'_DICT' if dict_items else ''}" - "}**") + var_header = (f"**${{{env_var_name}}}**") list_table_text = (f"{var_header}\n\n" ".. list-table::\n" From bf806beb7285a29f57be0b3a3a6f3160b4fbdb27 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 10:30:13 -0600 Subject: [PATCH 09/43] clean up formatting so it is easier to read the output --- metplus/util/doc_util.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/metplus/util/doc_util.py b/metplus/util/doc_util.py index 11e36f48e5..bd04399ac9 100755 --- a/metplus/util/doc_util.py +++ b/metplus/util/doc_util.py @@ -99,10 +99,12 @@ def print_doc_text(tool_name, met_var, dict_items): for item in dict_items: print(f' {item}') + print('\n==================================================\n') print(f'\n\nIn the {tool_name}_wrapper.py file, in the {wrapper_camel}Wrapper ' f'class, add the following to the WRAPPER_ENV_VAR_KEYS class ' f"variable list:\n\n\n '{env_var_name}',\n\n") + print('\n==================================================\n') print(f'In the create_c_dict function for {wrapper_camel}Wrapper, add a ' 'function call to read the new METplus config variables and save ' 'the value to be added to the wrapped MET config file.\n\n') @@ -110,7 +112,7 @@ def print_doc_text(tool_name, met_var, dict_items): print(f" self.add_met_config(name='{met_var}',\n" " data_type='',\n" f" metplus_configs=['{metplus_var}'])" - "\n\n" + "\n\n\n" "where can be string, list, int, float, bool, " "or thresh.\n\n") else: @@ -120,15 +122,19 @@ def print_doc_text(tool_name, met_var, dict_items): "function already exists for the item you are adding or to use " "as an example to write a new one.\n\n") + print('\n==================================================\n') print(f"In the parm/met_config/{wrapper_camel}Config_wrapped file, " "replace:\n\n") - print(f"{met_var} = ...\n\n with:\n\n//{met_var} =\n${{{env_var_name}}}\n\n") + print(f"{met_var} = ...\n\n with:\n\n//{met_var} =" + f"{' {' if dict_items else ''}\n${{{env_var_name}}}\n\n") + print('\n==================================================\n') print(f"\n\nIn docs/Users_Guide/wrappers.rst under {wrapper_camel} => " "METplus Configuration section, add:\n\n") for metplus_config_name in metplus_config_names: print(f'| :term:`{metplus_config_name}`') + print('\n==================================================\n') print(f"\n\nIn docs/Users_Guide/wrappers.rst under {wrapper_camel} => " "MET Configuration section, add:\n\n") var_header = (f"**${{{env_var_name}}}**") @@ -147,6 +153,7 @@ def print_doc_text(tool_name, met_var, dict_items): ) print(list_table_text) + print('\n==================================================\n') print(f"In docs/Users_Guide/glossary.rst, add:\n\n") for metplus_config_name, met_config_name in zip(metplus_config_names, met_config_values): glossary_entry = (f" {metplus_config_name}\n" From e12eebfbd49ab370bb18769250d27cafad7cd9d7 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 10:44:45 -0600 Subject: [PATCH 10/43] moved output_flag and nc_pairs_flag values from GRID_STAT_MET_CONFIG_OVERRIDES to new config variables, ci-run-diff --- ...DE_fcstFV3_obsGOES_BrightnessTempObjs.conf | 13 +++++++++++- ...GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf | 20 +++++++++++++++++-- ...GridStat_fcstHREFmean_obsStgIV_Gempak.conf | 6 +++++- ...GridStat_fcstHREFmean_obsStgIV_NetCDF.conf | 6 +++++- .../GridStat_fcstHRRR-TLE_obsStgIV_GRIB.conf | 10 ++++++++-- .../GridStat_fcstGloTEC_obsGloTEC_vx7.conf | 19 +++++++++++++++++- 6 files changed, 66 insertions(+), 8 deletions(-) diff --git a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf index 60d9efb9d0..f39d48cc50 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf @@ -75,7 +75,18 @@ GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped -GRID_STAT_MET_CONFIG_OVERRIDES = output_flag = {ctc = NONE;cts = NONE; eclv = NONE; nbrctc = BOTH; grad = NONE; dmap = BOTH; } nc_pairs_flag = {latlon = TRUE; raw = TRUE; diff = TRUE; distance_map = TRUE; apply_mask = TRUE;} +GRID_STAT_OUTPUT_FLAG_CTC = NONE +GRID_STAT_OUTPUT_FLAG_CTS = NONE +GRID_STAT_OUTPUT_FLAG_ECLV = NONE +GRID_STAT_OUTPUT_FLAG_NBRCTC = BOTH +GRID_STAT_OUTPUT_FLAG_GRAD = NONE +GRID_STAT_OUTPUT_FLAG_DMAP = BOTH + +GRID_STAT_NC_PAIRS_FLAG_LATLON = TRUE +GRID_STAT_NC_PAIRS_FLAG_RAW = TRUE +GRID_STAT_NC_PAIRS_FLAG_DIFF = TRUE +GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = TRUE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = TRUE GRID_STAT_OUTPUT_PREFIX = FV3_core_{instance} diff --git a/parm/use_cases/model_applications/cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf b/parm/use_cases/model_applications/cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf index bff936218e..ea9571bb71 100644 --- a/parm/use_cases/model_applications/cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf +++ b/parm/use_cases/model_applications/cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf @@ -68,8 +68,24 @@ GRID_STAT_MASK_GRID = #GRID_STAT_MASK_POLY = -GRID_STAT_MET_CONFIG_OVERRIDES = output_flag = {fho = STAT; cnt = STAT;sl1l2 = STAT;pct = STAT;pstd = STAT;eclv = NONE; nbrcnt = STAT; grad = NONE; } nc_pairs_flag = {latlon = TRUE; raw = TRUE; diff = TRUE; climo = TRUE; weight = FALSE; nbrhd = TRUE; fourier = FALSE; gradient = FALSE; apply_mask = TRUE;} - +GRID_STAT_OUTPUT_FLAG_FHO = STAT +GRID_STAT_OUTPUT_FLAG_CNT = STAT +GRID_STAT_OUTPUT_FLAG_SL1L2 = STAT +GRID_STAT_OUTPUT_FLAG_PCT = STAT +GRID_STAT_OUTPUT_FLAG_PSTD = STAT +GRID_STAT_OUTPUT_FLAG_ECLV = NONE +GRID_STAT_OUTPUT_FLAG_NBRCNT = STAT +GRID_STAT_OUTPUT_FLAG_GRAD = NONE + +GRID_STAT_NC_PAIRS_FLAG_LATLON = TRUE +GRID_STAT_NC_PAIRS_FLAG_RAW = TRUE +GRID_STAT_NC_PAIRS_FLAG_DIFF = TRUE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = TRUE +#GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE +GRID_STAT_NC_PAIRS_FLAG_NBRHD = TRUE +#GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE +#GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = TRUE #################################################################################################### # MODE Configurations diff --git a/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.conf b/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.conf index 250a7e5134..2eec451c7e 100644 --- a/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.conf +++ b/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.conf @@ -71,7 +71,11 @@ GRID_STAT_NEIGHBORHOOD_WIDTH = 3, 7, 15 GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 -GRID_STAT_MET_CONFIG_OVERRIDES = output_flag = { eclv = NONE; grad = NONE;dmap = STAT; } nc_pairs_flag = {distance_map = TRUE;} +GRID_STAT_OUTPUT_FLAG_ECLV = NONE +GRID_STAT_OUTPUT_FLAG_GRAD = NONE +GRID_STAT_OUTPUT_FLAG_DMAP = STAT + +GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = TRUE # HREF Mean Model Options: diff --git a/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.conf b/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.conf index db6ccd9f22..7466de085b 100644 --- a/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.conf +++ b/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.conf @@ -78,7 +78,11 @@ GRID_STAT_NEIGHBORHOOD_WIDTH = 3, 7, 15 GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 -GRID_STAT_MET_CONFIG_OVERRIDES = output_flag = { eclv = NONE; grad = NONE;dmap = STAT; } nc_pairs_flag = {distance_map = TRUE;} +GRID_STAT_OUTPUT_FLAG_ECLV = NONE +GRID_STAT_OUTPUT_FLAG_GRAD = NONE +GRID_STAT_OUTPUT_FLAG_DMAP = STAT + +GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = TRUE # HREF Mean Model Options: diff --git a/parm/use_cases/model_applications/precipitation/GridStat_fcstHRRR-TLE_obsStgIV_GRIB.conf b/parm/use_cases/model_applications/precipitation/GridStat_fcstHRRR-TLE_obsStgIV_GRIB.conf index 16f55cde37..836f563bf7 100644 --- a/parm/use_cases/model_applications/precipitation/GridStat_fcstHRRR-TLE_obsStgIV_GRIB.conf +++ b/parm/use_cases/model_applications/precipitation/GridStat_fcstHRRR-TLE_obsStgIV_GRIB.conf @@ -66,8 +66,14 @@ GRID_STAT_OUTPUT_PREFIX = PROB_{MODEL}_{CURRENT_FCST_NAME}_vs_{OBTYPE}_{CURRENT_ GRID_STAT_MASK_GRID = -GRID_STAT_MET_CONFIG_OVERRIDES = output_flag = {ctc = NONE; cts = NONE; pct = BOTH; pstd = BOTH; pjc = BOTH; prc = BOTH; eclv = STAT; grad = NONE;} - +GRID_STAT_OUTPUT_FLAG_CTC = NONE +GRID_STAT_OUTPUT_FLAG_CTS = NONE +GRID_STAT_OUTPUT_FLAG_PCT = BOTH +GRID_STAT_OUTPUT_FLAG_PSTD = BOTH +GRID_STAT_OUTPUT_FLAG_PJC = BOTH +GRID_STAT_OUTPUT_FLAG_PRC = BOTH +GRID_STAT_OUTPUT_FLAG_ECLV = STAT +GRID_STAT_OUTPUT_FLAG_GRAD = NONE # PHPT Model Options: diff --git a/parm/use_cases/model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.conf b/parm/use_cases/model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.conf index c368e4dbb5..e8fd578da9 100644 --- a/parm/use_cases/model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.conf +++ b/parm/use_cases/model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.conf @@ -88,7 +88,24 @@ LOOP_ORDER = times GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped # Override MET config file settings for this use case -GRID_STAT_MET_CONFIG_OVERRIDES = nc_pairs_flag = {latlon=TRUE; raw=TRUE; diff=TRUE;climo = FALSE;weight = FALSE; nbrhd = FALSE; fourier = FALSE;gradient = FALSE; apply_mask = FALSE;} output_flag = {mctc=STAT; mcts=STAT; cnt=STAT; sl1l2=STAT; eclv=NONE; grad=NONE;} file_type = NETCDF_NCCF; +GRID_STAT_MET_CONFIG_OVERRIDES = file_type = NETCDF_NCCF; + +GRID_STAT_OUTPUT_FLAG_MCTC = STAT +GRID_STAT_OUTPUT_FLAG_MCTS = STAT +GRID_STAT_OUTPUT_FLAG_CNT = STAT +GRID_STAT_OUTPUT_FLAG_SL1L2 = STAT +GRID_STAT_OUTPUT_FLAG_ECLV = NONE +GRID_STAT_OUTPUT_FLAG_GRAD = NONE + +GRID_STAT_NC_PAIRS_FLAG_LATLON = TRUE +GRID_STAT_NC_PAIRS_FLAG_RAW = TRUE +GRID_STAT_NC_PAIRS_FLAG_DIFF = TRUE +#GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +#GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE +#GRID_STAT_NC_PAIRS_FLAG_NBRHD = FALSE +#GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE +#GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE +#GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE # Name to identify model (forecast) data in output MODEL = GloTEC_without_cosmic From aa5405afe09fe7bb21b12fffcbc111242282c9ab Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 11:09:35 -0600 Subject: [PATCH 11/43] add env var to GridStat wrapped config, updated documentation for new config variables --- docs/Users_Guide/glossary.rst | 198 +++++++++++++++++++++++++ docs/Users_Guide/wrappers.rst | 153 ++++++++++++++++++- metplus/util/doc_util.py | 7 + parm/met_config/GridStatConfig_wrapped | 40 +---- 4 files changed, 361 insertions(+), 37 deletions(-) diff --git a/docs/Users_Guide/glossary.rst b/docs/Users_Guide/glossary.rst index 18e2a79cf3..890e51f973 100644 --- a/docs/Users_Guide/glossary.rst +++ b/docs/Users_Guide/glossary.rst @@ -5388,6 +5388,9 @@ METplus Configuration Glossary | *Used by:* EnsembleStat + ENSEMBLE_STAT_CLIMO_CDF_CDF_BINS + See :term:`ENSEMBLE_STAT_CLIMO_CDF_BINS` + ENSEMBLE_STAT_CLIMO_CDF_BINS Specify the value for 'climo_cdf.cdf_bins' in the MET configuration file for EnsembleStat. @@ -5993,3 +5996,198 @@ METplus Configuration Glossary | *Used by:* CyclonePlotter + GRID_STAT_CLIMO_CDF_CDF_BINS + See :term:`GRID_STAT_CLIMO_CDF_BINS` + + GRID_STAT_CLIMO_CDF_BINS + Specify the value for 'climo_cdf.cdf_bins' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_CLIMO_CDF_CENTER_BINS + Specify the value for 'climo_cdf.center_bins' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_CLIMO_CDF_WRITE_BINS + Specify the value for 'climo_cdf.write_bins' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + POINT_STAT_CLIMO_CDF_CDF_BINS + See :term:`POINT_STAT_CLIMO_CDF_BINS` + + POINT_STAT_CLIMO_CDF_BINS + Specify the value for 'climo_cdf.cdf_bins' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_CLIMO_CDF_CENTER_BINS + Specify the value for 'climo_cdf.center_bins' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + POINT_STAT_CLIMO_CDF_WRITE_BINS + Specify the value for 'climo_cdf.write_bins' in the MET configuration file for PointStat. + + | *Used by:* PointStat + + GRID_STAT_OUTPUT_FLAG_FHO + Specify the value for 'output_flag.fho' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_OUTPUT_FLAG_CTC + Specify the value for 'output_flag.ctc' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_OUTPUT_FLAG_CTS + Specify the value for 'output_flag.cts' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_OUTPUT_FLAG_MCTC + Specify the value for 'output_flag.mctc' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_OUTPUT_FLAG_MCTS + Specify the value for 'output_flag.mcts' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_OUTPUT_FLAG_CNT + Specify the value for 'output_flag.cnt' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_OUTPUT_FLAG_SL1L2 + Specify the value for 'output_flag.sl1l2' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_OUTPUT_FLAG_SAL1L2 + Specify the value for 'output_flag.sal1l2' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_OUTPUT_FLAG_VL1L2 + Specify the value for 'output_flag.vl1l2' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_OUTPUT_FLAG_VAL1L2 + Specify the value for 'output_flag.val1l2' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_OUTPUT_FLAG_VCNT + Specify the value for 'output_flag.vcnt' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_OUTPUT_FLAG_PCT + Specify the value for 'output_flag.pct' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_OUTPUT_FLAG_PSTD + Specify the value for 'output_flag.pstd' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_OUTPUT_FLAG_PJC + Specify the value for 'output_flag.pjc' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_OUTPUT_FLAG_PRC + Specify the value for 'output_flag.prc' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_OUTPUT_FLAG_ECLV + Specify the value for 'output_flag.eclv' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_OUTPUT_FLAG_NBRCTC + Specify the value for 'output_flag.nbrctc' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_OUTPUT_FLAG_NBRCTS + Specify the value for 'output_flag.nbrcts' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_OUTPUT_FLAG_NBRCNT + Specify the value for 'output_flag.nbrcnt' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_OUTPUT_FLAG_GRAD + Specify the value for 'output_flag.grad' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_OUTPUT_FLAG_DMAP + Specify the value for 'output_flag.dmap' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_NC_PAIRS_FLAG_LATLON + Specify the value for 'nc_pairs_flag.latlon' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_NC_PAIRS_FLAG_RAW + Specify the value for 'nc_pairs_flag.raw' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_NC_PAIRS_FLAG_DIFF + Specify the value for 'nc_pairs_flag.diff' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_NC_PAIRS_FLAG_CLIMO + Specify the value for 'nc_pairs_flag.climo' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_NC_PAIRS_FLAG_CLIMO_CDP + Specify the value for 'nc_pairs_flag.climo_cdp' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_NC_PAIRS_FLAG_WEIGHT + Specify the value for 'nc_pairs_flag.weight' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_NC_PAIRS_FLAG_NBRHD + Specify the value for 'nc_pairs_flag.nbrhd' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_NC_PAIRS_FLAG_FOURIER + Specify the value for 'nc_pairs_flag.fourier' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_NC_PAIRS_FLAG_GRADIENT + Specify the value for 'nc_pairs_flag.gradient' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP + Specify the value for 'nc_pairs_flag.distance_map' in the MET configuration file for GridStat. + + | *Used by:* GridStat + + GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK + Specify the value for 'nc_pairs_flag.apply_mask' in the MET configuration file for GridStat. + + | *Used by:* GridStat diff --git a/docs/Users_Guide/wrappers.rst b/docs/Users_Guide/wrappers.rst index 16fc09561a..29a704ce35 100755 --- a/docs/Users_Guide/wrappers.rst +++ b/docs/Users_Guide/wrappers.rst @@ -1047,7 +1047,7 @@ Below the file contents are descriptions of each environment variable referenced * - :term:`GRID_DIAG_MASK_POLY` - mask.poly -.. note:: Since the default value in the MET config file for 'grid' is grid = [ "FULL" ];, setting GRID_STAT_MASK_GRID to an empty string will result in a value of grid = []; in the MET config file. +.. note:: Since the default value in the MET config file for 'grid' is grid = [ "FULL" ];, setting GRID_DIAG_MASK_GRID to an empty string will result in a value of grid = []; in the MET config file. **${METPLUS_MET_CONFIG_OVERRIDES}** @@ -1098,6 +1098,41 @@ METplus Configuration | :term:`GRID_STAT_REGRID_WIDTH` | :term:`GRID_STAT_REGRID_VLD_THRESH` | :term:`GRID_STAT_REGRID_SHAPE` +| :term:`GRID_STAT_CLIMO_CDF_BINS` +| :term:`GRID_STAT_CLIMO_CDF_CENTER_BINS` +| :term:`GRID_STAT_CLIMO_CDF_WRITE_BINS` +| :term:`GRID_STAT_OUTPUT_FLAG_FHO` +| :term:`GRID_STAT_OUTPUT_FLAG_CTC` +| :term:`GRID_STAT_OUTPUT_FLAG_CTS` +| :term:`GRID_STAT_OUTPUT_FLAG_MCTC` +| :term:`GRID_STAT_OUTPUT_FLAG_MCTS` +| :term:`GRID_STAT_OUTPUT_FLAG_CNT` +| :term:`GRID_STAT_OUTPUT_FLAG_SL1L2` +| :term:`GRID_STAT_OUTPUT_FLAG_SAL1L2` +| :term:`GRID_STAT_OUTPUT_FLAG_VL1L2` +| :term:`GRID_STAT_OUTPUT_FLAG_VAL1L2` +| :term:`GRID_STAT_OUTPUT_FLAG_VCNT` +| :term:`GRID_STAT_OUTPUT_FLAG_PCT` +| :term:`GRID_STAT_OUTPUT_FLAG_PSTD` +| :term:`GRID_STAT_OUTPUT_FLAG_PJC` +| :term:`GRID_STAT_OUTPUT_FLAG_PRC` +| :term:`GRID_STAT_OUTPUT_FLAG_ECLV` +| :term:`GRID_STAT_OUTPUT_FLAG_NBRCTC` +| :term:`GRID_STAT_OUTPUT_FLAG_NBRCTS` +| :term:`GRID_STAT_OUTPUT_FLAG_NBRCNT` +| :term:`GRID_STAT_OUTPUT_FLAG_GRAD` +| :term:`GRID_STAT_OUTPUT_FLAG_DMAP` +| :term:`GRID_STAT_NC_PAIRS_FLAG_LATLON` +| :term:`GRID_STAT_NC_PAIRS_FLAG_RAW` +| :term:`GRID_STAT_NC_PAIRS_FLAG_DIFF` +| :term:`GRID_STAT_NC_PAIRS_FLAG_CLIMO` +| :term:`GRID_STAT_NC_PAIRS_FLAG_CLIMO_CDP` +| :term:`GRID_STAT_NC_PAIRS_FLAG_WEIGHT` +| :term:`GRID_STAT_NC_PAIRS_FLAG_NBRHD` +| :term:`GRID_STAT_NC_PAIRS_FLAG_FOURIER` +| :term:`GRID_STAT_NC_PAIRS_FLAG_GRADIENT` +| :term:`GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP` +| :term:`GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK` | :term:`GRID_STAT_MASK_GRID` (optional) | :term:`GRID_STAT_MASK_POLY` (optional) | :term:`GRID_STAT_MET_CONFIG_OVERRIDES` @@ -1327,6 +1362,104 @@ Below the file contents are descriptions of each environment variable referenced * - :term:`GRID_STAT_MET_CONFIG_OVERRIDES` - n/a +**${METPLUS_CLIMO_CDF_DICT}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`GRID_STAT_CLIMO_CDF_BINS` + - climo_cdf.cdf_bins + * - :term:`GRID_STAT_CLIMO_CDF_CENTER_BINS` + - climo_cdf.center_bins + * - :term:`GRID_STAT_CLIMO_CDF_WRITE_BINS` + - climo_cdf.write_bins + +**${METPLUS_OUTPUT_FLAG_DICT}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`GRID_STAT_OUTPUT_FLAG_FHO` + - output_flag.fho + * - :term:`GRID_STAT_OUTPUT_FLAG_CTC` + - output_flag.ctc + * - :term:`GRID_STAT_OUTPUT_FLAG_CTS` + - output_flag.cts + * - :term:`GRID_STAT_OUTPUT_FLAG_MCTC` + - output_flag.mctc + * - :term:`GRID_STAT_OUTPUT_FLAG_MCTS` + - output_flag.mcts + * - :term:`GRID_STAT_OUTPUT_FLAG_CNT` + - output_flag.cnt + * - :term:`GRID_STAT_OUTPUT_FLAG_SL1L2` + - output_flag.sl1l2 + * - :term:`GRID_STAT_OUTPUT_FLAG_SAL1L2` + - output_flag.sal1l2 + * - :term:`GRID_STAT_OUTPUT_FLAG_VL1L2` + - output_flag.vl1l2 + * - :term:`GRID_STAT_OUTPUT_FLAG_VAL1L2` + - output_flag.val1l2 + * - :term:`GRID_STAT_OUTPUT_FLAG_VCNT` + - output_flag.vcnt + * - :term:`GRID_STAT_OUTPUT_FLAG_PCT` + - output_flag.pct + * - :term:`GRID_STAT_OUTPUT_FLAG_PSTD` + - output_flag.pstd + * - :term:`GRID_STAT_OUTPUT_FLAG_PJC` + - output_flag.pjc + * - :term:`GRID_STAT_OUTPUT_FLAG_PRC` + - output_flag.prc + * - :term:`GRID_STAT_OUTPUT_FLAG_ECLV` + - output_flag.eclv + * - :term:`GRID_STAT_OUTPUT_FLAG_NBRCTC` + - output_flag.nbrctc + * - :term:`GRID_STAT_OUTPUT_FLAG_NBRCTS` + - output_flag.nbrcts + * - :term:`GRID_STAT_OUTPUT_FLAG_NBRCNT` + - output_flag.nbrcnt + * - :term:`GRID_STAT_OUTPUT_FLAG_GRAD` + - output_flag.grad + * - :term:`GRID_STAT_OUTPUT_FLAG_DMAP` + - output_flag.dmap + +**${METPLUS_NC_PAIRS_FLAG_DICT}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`GRID_STAT_NC_PAIRS_FLAG_LATLON` + - nc_pairs_flag.latlon + * - :term:`GRID_STAT_NC_PAIRS_FLAG_RAW` + - nc_pairs_flag.raw + * - :term:`GRID_STAT_NC_PAIRS_FLAG_DIFF` + - nc_pairs_flag.diff + * - :term:`GRID_STAT_NC_PAIRS_FLAG_CLIMO` + - nc_pairs_flag.climo + * - :term:`GRID_STAT_NC_PAIRS_FLAG_CLIMO_CDP` + - nc_pairs_flag.climo_cdp + * - :term:`GRID_STAT_NC_PAIRS_FLAG_WEIGHT` + - nc_pairs_flag.weight + * - :term:`GRID_STAT_NC_PAIRS_FLAG_NBRHD` + - nc_pairs_flag.nbrhd + * - :term:`GRID_STAT_NC_PAIRS_FLAG_FOURIER` + - nc_pairs_flag.fourier + * - :term:`GRID_STAT_NC_PAIRS_FLAG_GRADIENT` + - nc_pairs_flag.gradient + * - :term:`GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP` + - nc_pairs_flag.distance_map + * - :term:`GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK` + - nc_pairs_flag.apply_mask + + .. _make_plots_wrapper: MakePlots @@ -2369,6 +2502,9 @@ Configuration | :term:`POINT_STAT_SKIP_IF_OUTPUT_EXISTS` | :term:`POINT_STAT_DESC` | :term:`POINT_STAT_MET_CONFIG_OVERRIDES` +| :term:`POINT_STAT_CLIMO_CDF_BINS` +| :term:`POINT_STAT_CLIMO_CDF_CENTER_BINS` +| :term:`POINT_STAT_CLIMO_CDF_WRITE_BINS` | :term:`FCST_POINT_STAT_WINDOW_BEGIN` (optional) | :term:`FCST_POINT_STAT_WINDOW_END` (optional) | :term:`OBS_POINT_STAT_WINDOW_BEGIN` (optional) @@ -2601,6 +2737,21 @@ Below the file contents are descriptions of each environment variable referenced * - :term:`POINT_STAT_MET_CONFIG_OVERRIDES` - n/a +**${METPLUS_CLIMO_CDF_DICT}** + +.. list-table:: + :widths: 5 5 + :header-rows: 0 + + * - METplus Config(s) + - MET Config File + * - :term:`POINT_STAT_CLIMO_CDF_BINS` + - climo_cdf.cdf_bins + * - :term:`POINT_STAT_CLIMO_CDF_CENTER_BINS` + - climo_cdf.center_bins + * - :term:`POINT_STAT_CLIMO_CDF_WRITE_BINS` + - climo_cdf.write_bins + .. _py_embed_ingest_wrapper: PyEmbedIngest diff --git a/metplus/util/doc_util.py b/metplus/util/doc_util.py index bd04399ac9..64648b9416 100755 --- a/metplus/util/doc_util.py +++ b/metplus/util/doc_util.py @@ -123,6 +123,13 @@ def print_doc_text(tool_name, met_var, dict_items): "as an example to write a new one.\n\n") print('\n==================================================\n') + print('Add the new variables to the basic use case example for the tool,\n' + f'i.e. parm/use_cases/met_tool_wrapper/{wrapper_camel}/' + f'{wrapper_camel}.conf:\n\n') + for mp_config in metplus_config_names: + print(f'#{mp_config} =') + + print('\n\n==================================================\n') print(f"In the parm/met_config/{wrapper_camel}Config_wrapped file, " "replace:\n\n") print(f"{met_var} = ...\n\n with:\n\n//{met_var} =" diff --git a/parm/met_config/GridStatConfig_wrapped b/parm/met_config/GridStatConfig_wrapped index b916d0f3b8..3333a8f332 100644 --- a/parm/met_config/GridStatConfig_wrapped +++ b/parm/met_config/GridStatConfig_wrapped @@ -193,47 +193,15 @@ distance_map = { // // Statistical output types // -output_flag = { - fho = NONE; - ctc = STAT; - cts = STAT; - mctc = NONE; - mcts = NONE; - cnt = NONE; - sl1l2 = NONE; - sal1l2 = NONE; - vl1l2 = NONE; - val1l2 = NONE; - vcnt = NONE; - pct = NONE; - pstd = NONE; - pjc = NONE; - prc = NONE; - eclv = BOTH; - nbrctc = NONE; - nbrcts = NONE; - nbrcnt = NONE; - grad = BOTH; - dmap = NONE; -} +//output_flag = { +${METPLUS_OUTPUT_FLAG_DICT} // // NetCDF matched pairs output file // May be set separately in each "obs.field" entry // -nc_pairs_flag = { - latlon = FALSE; - raw = FALSE; - diff = FALSE; - climo = FALSE; - climo_cdp = FALSE; - weight = FALSE; - nbrhd = FALSE; - fourier = FALSE; - gradient = FALSE; - distance_map = FALSE; - apply_mask = FALSE; -} +// nc_pairs_flag = { +${METPLUS_NC_PAIRS_FLAG_DICT} //////////////////////////////////////////////////////////////////////////////// From e73fcb442ade9969355566644f89cdd9354ce2cc Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 11:10:20 -0600 Subject: [PATCH 12/43] trigger diff tests with ci-run-diff --- metplus/util/doc_util.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metplus/util/doc_util.py b/metplus/util/doc_util.py index 64648b9416..4c53b6ece6 100755 --- a/metplus/util/doc_util.py +++ b/metplus/util/doc_util.py @@ -70,7 +70,6 @@ def print_doc_text(tool_name, met_var, dict_items): wrapper_camel = get_wrapper_name(wrapper_caps) - metplus_var = f'{wrapper_caps}_{met_var_caps}' metplus_config_names = [] From cb3c38bdc6ecf7512cfb035105e21feedc0ce2a5 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 11:55:14 -0600 Subject: [PATCH 13/43] changed output_flag and nc_pairs_flag settings to match previous default settings of the wrapped GridStatConfig file in parm/met_config --- .../met_tool_wrapper/GridStat/GridStat.conf | 18 +++++++++--------- .../GridStat/GridStat_python_embedding.conf | 12 ++++++++++++ .../GridStat_fcstCESM_obsGFS_ConusTemp.conf | 8 ++++++-- ...tat_fcstFV3_obsGOES_BrightnessTempDmap.conf | 10 ++-------- ...cstHRRR_obsPracPerfect_SurrogateSevere.conf | 8 ++++++-- ...RRR_obsPracPerfect_SurrogateSevereProb.conf | 4 ---- ...ODE_fcstFV3_obsGOES_BrightnessTempObjs.conf | 8 -------- .../GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf | 12 ++---------- ...GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf | 10 ++++++---- ...at_fcstGFS_obsGFS_climoNCEP_MultiField.conf | 10 ++++++---- .../GridStat_fcstGFS_obsCCPA_GRIB.conf | 10 +++++++--- .../GridStat_fcstHREFmean_obsStgIV_Gempak.conf | 11 +++++++++-- .../GridStat_fcstHREFmean_obsStgIV_NetCDF.conf | 11 +++++++++-- .../GridStat_fcstHRRR-TLE_obsStgIV_GRIB.conf | 9 ++++++--- ...ysis_fcstNMME_obsCPC_seasonal_forecast.conf | 9 ++------- .../GridStat_fcstGloTEC_obsGloTEC_vx7.conf | 17 +++++------------ 16 files changed, 87 insertions(+), 80 deletions(-) diff --git a/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf b/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf index 61154d8700..dcca51a775 100644 --- a/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf +++ b/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf @@ -137,8 +137,8 @@ GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_vs_{OBTYPE}_{CURRENT_OBS_N #GRID_STAT_CLIMO_CDF_WRITE_BINS = True #GRID_STAT_OUTPUT_FLAG_FHO = NONE -#GRID_STAT_OUTPUT_FLAG_CTC = STAT -#GRID_STAT_OUTPUT_FLAG_CTS = STAT +GRID_STAT_OUTPUT_FLAG_CTC = STAT +GRID_STAT_OUTPUT_FLAG_CTS = STAT #GRID_STAT_OUTPUT_FLAG_MCTC = NONE #GRID_STAT_OUTPUT_FLAG_MCTS = NONE #GRID_STAT_OUTPUT_FLAG_CNT = NONE @@ -151,24 +151,24 @@ GRID_STAT_OUTPUT_PREFIX = {MODEL}_{CURRENT_FCST_NAME}_vs_{OBTYPE}_{CURRENT_OBS_N #GRID_STAT_OUTPUT_FLAG_PSTD = NONE #GRID_STAT_OUTPUT_FLAG_PJC = NONE #GRID_STAT_OUTPUT_FLAG_PRC = NONE -#GRID_STAT_OUTPUT_FLAG_ECLV = BOTH +GRID_STAT_OUTPUT_FLAG_ECLV = BOTH #GRID_STAT_OUTPUT_FLAG_NBRCTC = NONE #GRID_STAT_OUTPUT_FLAG_NBRCTS = NONE #GRID_STAT_OUTPUT_FLAG_NBRCNT = NONE -#GRID_STAT_OUTPUT_FLAG_GRAD = BOTH +GRID_STAT_OUTPUT_FLAG_GRAD = BOTH #GRID_STAT_OUTPUT_FLAG_DMAP = NONE -#GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE -#GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE -#GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE -#GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE #GRID_STAT_NC_PAIRS_FLAG_CLIMO_CDP = FALSE #GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE #GRID_STAT_NC_PAIRS_FLAG_NBRHD = FALSE #GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE #GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE #GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE -#GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE # End of [config] section and start of [dir] section [dir] diff --git a/parm/use_cases/met_tool_wrapper/GridStat/GridStat_python_embedding.conf b/parm/use_cases/met_tool_wrapper/GridStat/GridStat_python_embedding.conf index 8440b6ef78..156d0509f7 100644 --- a/parm/use_cases/met_tool_wrapper/GridStat/GridStat_python_embedding.conf +++ b/parm/use_cases/met_tool_wrapper/GridStat/GridStat_python_embedding.conf @@ -115,6 +115,18 @@ OBS_GRID_STAT_PROB_THRESH = ==0.1 GRID_STAT_OUTPUT_PREFIX = {MODEL}_vs_{OBTYPE} +GRID_STAT_OUTPUT_FLAG_CTC = STAT +GRID_STAT_OUTPUT_FLAG_CTS = STAT +GRID_STAT_OUTPUT_FLAG_ECLV = BOTH +GRID_STAT_OUTPUT_FLAG_GRAD = BOTH + +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE + + # End of [config] section and start of [dir] section [dir] diff --git a/parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf b/parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf index dc2d27fc97..6001aae440 100644 --- a/parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf +++ b/parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf @@ -82,8 +82,12 @@ GRID_STAT_OUTPUT_PREFIX={MODEL}_{CURRENT_OBS_NAME}_vs_{OBTYPE} GRID_STAT_OUTPUT_FLAG_CNT = STAT GRID_STAT_OUTPUT_FLAG_SL1L2 = STAT -GRID_STAT_OUTPUT_FLAG_ECLV = NONE -GRID_STAT_OUTPUT_FLAG_GRAD = NONE + +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE GRID_STAT_MET_CONFIG_OVERRIDES = interp = { field = NONE;} diff --git a/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf b/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf index 9e3c4bca65..98e0430461 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf @@ -40,17 +40,11 @@ GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped GRID_STAT_OUTPUT_PREFIX = FV3_core_{instance} -GRID_STAT_OUTPUT_FLAG_CTC = NONE -GRID_STAT_OUTPUT_FLAG_CTS = NONE -GRID_STAT_OUTPUT_FLAG_ECLV = NONE -GRID_STAT_OUTPUT_FLAG_GRAD = NONE GRID_STAT_OUTPUT_FLAG_DMAP = DMAP -GRID_STAT_NC_PAIRS_FLAG_LATLON = TRUE -GRID_STAT_NC_PAIRS_FLAG_RAW = TRUE -GRID_STAT_NC_PAIRS_FLAG_DIFF = TRUE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = TRUE -GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = TRUE + [dir] # Input and Output Diretory of the object data diff --git a/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.conf b/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.conf index 12bb5264a5..27aff82488 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevere.conf @@ -82,8 +82,12 @@ GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped GRID_STAT_OUTPUT_FLAG_CTC = BOTH GRID_STAT_OUTPUT_FLAG_CTS = BOTH -GRID_STAT_OUTPUT_FLAG_ECLV = NONE -GRID_STAT_OUTPUT_FLAG_GRAD = NONE + +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE [dir] diff --git a/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.conf b/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.conf index 80d4002ccc..3d3aa4085e 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.conf @@ -83,14 +83,10 @@ FCST_GRID_STAT_INPUT_DATATYPE = NETCDF GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped -GRID_STAT_OUTPUT_FLAG_CTC = NONE -GRID_STAT_OUTPUT_FLAG_CTS = NONE GRID_STAT_OUTPUT_FLAG_PCT = BOTH GRID_STAT_OUTPUT_FLAG_PSTD = BOTH GRID_STAT_OUTPUT_FLAG_PJC = BOTH GRID_STAT_OUTPUT_FLAG_PRC = BOTH -GRID_STAT_OUTPUT_FLAG_ECLV = NONE -GRID_STAT_OUTPUT_FLAG_GRAD = NONE [dir] diff --git a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf index f39d48cc50..0196ca8227 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/MODE_fcstFV3_obsGOES_BrightnessTempObjs.conf @@ -75,18 +75,10 @@ GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped -GRID_STAT_OUTPUT_FLAG_CTC = NONE -GRID_STAT_OUTPUT_FLAG_CTS = NONE -GRID_STAT_OUTPUT_FLAG_ECLV = NONE GRID_STAT_OUTPUT_FLAG_NBRCTC = BOTH -GRID_STAT_OUTPUT_FLAG_GRAD = NONE GRID_STAT_OUTPUT_FLAG_DMAP = BOTH -GRID_STAT_NC_PAIRS_FLAG_LATLON = TRUE -GRID_STAT_NC_PAIRS_FLAG_RAW = TRUE -GRID_STAT_NC_PAIRS_FLAG_DIFF = TRUE GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = TRUE -GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = TRUE GRID_STAT_OUTPUT_PREFIX = FV3_core_{instance} diff --git a/parm/use_cases/model_applications/cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf b/parm/use_cases/model_applications/cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf index ea9571bb71..d604f85b40 100644 --- a/parm/use_cases/model_applications/cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf +++ b/parm/use_cases/model_applications/cryosphere/GridStat_MODE_fcstIMS_obsNCEP_sea_ice.conf @@ -68,24 +68,16 @@ GRID_STAT_MASK_GRID = #GRID_STAT_MASK_POLY = +GRID_STAT_OUTPUT_FLAG_CTC = STAT +GRID_STAT_OUTPUT_FLAG_CTS = STAT GRID_STAT_OUTPUT_FLAG_FHO = STAT GRID_STAT_OUTPUT_FLAG_CNT = STAT GRID_STAT_OUTPUT_FLAG_SL1L2 = STAT GRID_STAT_OUTPUT_FLAG_PCT = STAT GRID_STAT_OUTPUT_FLAG_PSTD = STAT -GRID_STAT_OUTPUT_FLAG_ECLV = NONE GRID_STAT_OUTPUT_FLAG_NBRCNT = STAT -GRID_STAT_OUTPUT_FLAG_GRAD = NONE -GRID_STAT_NC_PAIRS_FLAG_LATLON = TRUE -GRID_STAT_NC_PAIRS_FLAG_RAW = TRUE -GRID_STAT_NC_PAIRS_FLAG_DIFF = TRUE -GRID_STAT_NC_PAIRS_FLAG_CLIMO = TRUE -#GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE GRID_STAT_NC_PAIRS_FLAG_NBRHD = TRUE -#GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE -#GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE -GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = TRUE #################################################################################################### # MODE Configurations diff --git a/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf b/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf index ec97313002..2fffb058c0 100644 --- a/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf +++ b/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_Sfc_MultiField.conf @@ -131,11 +131,13 @@ GRID_STAT_MASK_POLY = {INPUT_BASE}/model_applications/medium_range/poly/NHX.nc, GRID_STAT_CLIMO_CDF_WRITE_BINS = False -GRID_STAT_OUTPUT_FLAG_CTC = NONE -GRID_STAT_OUTPUT_FLAG_CTS = NONE GRID_STAT_OUTPUT_FLAG_SL1L2 = STAT -GRID_STAT_OUTPUT_FLAG_ECLV = NONE -GRID_STAT_OUTPUT_FLAG_GRAD = NONE + +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE GRID_STAT_MET_CONFIG_OVERRIDES = climo_mean = { regrid = { method = BILIN;width = 2;} day_interval = 1; time_interp_method = NEAREST;} grid_weight_flag = COS_LAT; diff --git a/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf b/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf index 3d8efe55c4..4ee8ccafc4 100644 --- a/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf +++ b/parm/use_cases/model_applications/medium_range/GridStat_fcstGFS_obsGFS_climoNCEP_MultiField.conf @@ -61,12 +61,14 @@ GRID_STAT_MASK_POLY = {INPUT_BASE}/model_applications/medium_range/poly/NHX.nc, GRID_STAT_CLIMO_CDF_WRITE_BINS = False -GRID_STAT_OUTPUT_FLAG_CTC = NONE -GRID_STAT_OUTPUT_FLAG_CTS = NONE GRID_STAT_OUTPUT_FLAG_SAL1L2 = STAT GRID_STAT_OUTPUT_FLAG_VAL1L2 = STAT -GRID_STAT_OUTPUT_FLAG_ECLV = NONE -GRID_STAT_OUTPUT_FLAG_GRAD = NONE + +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE GRID_STAT_MET_CONFIG_OVERRIDES = climo_mean = { regrid = { method = BILIN;width = 2;} day_interval = 1; time_interp_method = NEAREST;} grid_weight_flag = COS_LAT; climo_mean = fcst; diff --git a/parm/use_cases/model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.conf b/parm/use_cases/model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.conf index 3641e9edff..b4a7d9a045 100644 --- a/parm/use_cases/model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.conf +++ b/parm/use_cases/model_applications/precipitation/GridStat_fcstGFS_obsCCPA_GRIB.conf @@ -58,9 +58,13 @@ GRID_STAT_MASK_POLY = {INPUT_BASE}/model_applications/precipitation/poly/CONUS.n GRID_STAT_CLIMO_CDF_WRITE_BINS = False -GRID_STAT_OUTPUT_FLAG_CTS = NONE -GRID_STAT_OUTPUT_FLAG_ECLV = NONE -GRID_STAT_OUTPUT_FLAG_GRAD = NONE +GRID_STAT_OUTPUT_FLAG_CTC = STAT + +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE GRID_STAT_MET_CONFIG_OVERRIDES = climo_mean = { regrid = { method = BILIN;width = 2;} time_interp_method = NEAREST;} diff --git a/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.conf b/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.conf index 2eec451c7e..f42ac929a1 100644 --- a/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.conf +++ b/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.conf @@ -71,10 +71,17 @@ GRID_STAT_NEIGHBORHOOD_WIDTH = 3, 7, 15 GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 -GRID_STAT_OUTPUT_FLAG_ECLV = NONE -GRID_STAT_OUTPUT_FLAG_GRAD = NONE +GRID_STAT_OUTPUT_FLAG_CTC = STAT +GRID_STAT_OUTPUT_FLAG_CTS = STAT +GRID_STAT_OUTPUT_FLAG_ECLV = BOTH +GRID_STAT_OUTPUT_FLAG_GRAD = BOTH GRID_STAT_OUTPUT_FLAG_DMAP = STAT +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = TRUE # HREF Mean Model Options: diff --git a/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.conf b/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.conf index 7466de085b..9b8d50df74 100644 --- a/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.conf +++ b/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.conf @@ -78,10 +78,17 @@ GRID_STAT_NEIGHBORHOOD_WIDTH = 3, 7, 15 GRID_STAT_NEIGHBORHOOD_SHAPE = SQUARE GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 -GRID_STAT_OUTPUT_FLAG_ECLV = NONE -GRID_STAT_OUTPUT_FLAG_GRAD = NONE +GRID_STAT_OUTPUT_FLAG_CTC = STAT +GRID_STAT_OUTPUT_FLAG_CTS = STAT +GRID_STAT_OUTPUT_FLAG_ECLV = BOTH +GRID_STAT_OUTPUT_FLAG_GRAD = BOTH GRID_STAT_OUTPUT_FLAG_DMAP = STAT +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = TRUE # HREF Mean Model Options: diff --git a/parm/use_cases/model_applications/precipitation/GridStat_fcstHRRR-TLE_obsStgIV_GRIB.conf b/parm/use_cases/model_applications/precipitation/GridStat_fcstHRRR-TLE_obsStgIV_GRIB.conf index 836f563bf7..8a78237763 100644 --- a/parm/use_cases/model_applications/precipitation/GridStat_fcstHRRR-TLE_obsStgIV_GRIB.conf +++ b/parm/use_cases/model_applications/precipitation/GridStat_fcstHRRR-TLE_obsStgIV_GRIB.conf @@ -66,14 +66,17 @@ GRID_STAT_OUTPUT_PREFIX = PROB_{MODEL}_{CURRENT_FCST_NAME}_vs_{OBTYPE}_{CURRENT_ GRID_STAT_MASK_GRID = -GRID_STAT_OUTPUT_FLAG_CTC = NONE -GRID_STAT_OUTPUT_FLAG_CTS = NONE GRID_STAT_OUTPUT_FLAG_PCT = BOTH GRID_STAT_OUTPUT_FLAG_PSTD = BOTH GRID_STAT_OUTPUT_FLAG_PJC = BOTH GRID_STAT_OUTPUT_FLAG_PRC = BOTH GRID_STAT_OUTPUT_FLAG_ECLV = STAT -GRID_STAT_OUTPUT_FLAG_GRAD = NONE + +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE # PHPT Model Options: diff --git a/parm/use_cases/model_applications/s2s/GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.conf b/parm/use_cases/model_applications/s2s/GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.conf index 3672a5b56c..c351b7efe7 100644 --- a/parm/use_cases/model_applications/s2s/GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.conf +++ b/parm/use_cases/model_applications/s2s/GridStat_SeriesAnalysis_fcstNMME_obsCPC_seasonal_forecast.conf @@ -53,16 +53,11 @@ OBTYPE = CPC # location of grid_stat MET config file GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped -GRID_STAT_OUTPUT_FLAG_CTS = NONE +GRID_STAT_OUTPUT_FLAG_CTC = STAT GRID_STAT_OUTPUT_FLAG_CNT = STAT GRID_STAT_OUTPUT_FLAG_SL1L2 = STAT -GRID_STAT_OUTPUT_FLAG_ECLV = NONE -GRID_STAT_OUTPUT_FLAG_GRAD = NONE -GRID_STAT_NC_PAIRS_FLAG_LATLON = TRUE -GRID_STAT_NC_PAIRS_FLAG_RAW = TRUE -GRID_STAT_NC_PAIRS_FLAG_DIFF = TRUE -GRID_STAT_NC_PAIRS_FLAG_CLIMO = TRUE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE # Override MET config file settings GRID_STAT_MET_CONFIG_OVERRIDES = nc_pairs_var_name = "precip"; diff --git a/parm/use_cases/model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.conf b/parm/use_cases/model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.conf index e8fd578da9..c0b106b956 100644 --- a/parm/use_cases/model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.conf +++ b/parm/use_cases/model_applications/space_weather/GridStat_fcstGloTEC_obsGloTEC_vx7.conf @@ -90,22 +90,15 @@ GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped # Override MET config file settings for this use case GRID_STAT_MET_CONFIG_OVERRIDES = file_type = NETCDF_NCCF; +GRID_STAT_OUTPUT_FLAG_CTC = STAT +GRID_STAT_OUTPUT_FLAG_CTS = STAT GRID_STAT_OUTPUT_FLAG_MCTC = STAT GRID_STAT_OUTPUT_FLAG_MCTS = STAT GRID_STAT_OUTPUT_FLAG_CNT = STAT GRID_STAT_OUTPUT_FLAG_SL1L2 = STAT -GRID_STAT_OUTPUT_FLAG_ECLV = NONE -GRID_STAT_OUTPUT_FLAG_GRAD = NONE - -GRID_STAT_NC_PAIRS_FLAG_LATLON = TRUE -GRID_STAT_NC_PAIRS_FLAG_RAW = TRUE -GRID_STAT_NC_PAIRS_FLAG_DIFF = TRUE -#GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE -#GRID_STAT_NC_PAIRS_FLAG_WEIGHT = FALSE -#GRID_STAT_NC_PAIRS_FLAG_NBRHD = FALSE -#GRID_STAT_NC_PAIRS_FLAG_FOURIER = FALSE -#GRID_STAT_NC_PAIRS_FLAG_GRADIENT = FALSE -#GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE + +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE # Name to identify model (forecast) data in output MODEL = GloTEC_without_cosmic From e4294b7d5cb33a4b93f8f78587954393131a827e Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 11:55:41 -0600 Subject: [PATCH 14/43] ci-run-diff --- parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf b/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf index dcca51a775..3d41b6b99b 100644 --- a/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf +++ b/parm/use_cases/met_tool_wrapper/GridStat/GridStat.conf @@ -170,6 +170,7 @@ GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE #GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = FALSE GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE + # End of [config] section and start of [dir] section [dir] From 4ca0bb6c824ab511dcc9dcf43876d5e0c2e67c2c Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 12:01:39 -0600 Subject: [PATCH 15/43] fixed typo in value for DMAP --- .../GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf b/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf index 98e0430461..aab5494da8 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf @@ -40,7 +40,7 @@ GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped GRID_STAT_OUTPUT_PREFIX = FV3_core_{instance} -GRID_STAT_OUTPUT_FLAG_DMAP = DMAP +GRID_STAT_OUTPUT_FLAG_DMAP = BOTH GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = TRUE From b7abcda6790144619067ec337bf8491f73a1529a Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 12:12:53 -0600 Subject: [PATCH 16/43] only copy log directory to error_logs if there is an error in the use case, ci-run-diff --- ci/jobs/run_use_cases_docker.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ci/jobs/run_use_cases_docker.py b/ci/jobs/run_use_cases_docker.py index 1cd29ce9be..05b395dbab 100755 --- a/ci/jobs/run_use_cases_docker.py +++ b/ci/jobs/run_use_cases_docker.py @@ -29,6 +29,17 @@ def copy_error_logs(): 'logs') if not os.path.isdir(log_dir): continue + + # check if there are errors in the metplus.log file and + # only copy directory if there are any errors + metplus_log = os.path.join(log_dir, 'metplus.log') + found_errors = False + with open(metplus_log, 'r') as file_handle: + if 'ERROR:' in file_handle.read(): + found_errors = True + if not found_errors: + continue + output_dir = os.path.join(ERROR_LOG_DIR, use_case_dir) log_files = os.listdir(log_dir) From 8ca3e4be2fbba82025e8643f1b6ae89b861e2a40 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 13:06:38 -0600 Subject: [PATCH 17/43] try to using image diff logic for pdf files --- ci/util/diff_util.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/util/diff_util.py b/ci/util/diff_util.py index c969500a0a..6d14257e8b 100644 --- a/ci/util/diff_util.py +++ b/ci/util/diff_util.py @@ -8,6 +8,7 @@ '.png', '.jpg', '.jpeg', + '.pdf', ] NETCDF_EXTENSIONS = [ From 9c5a524d1044c5bbf6e15c9983d6fbd50c3a91af Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 13:07:13 -0600 Subject: [PATCH 18/43] fix output flags to match previous settings, ci-run-diff --- .../climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf | 3 +++ ...ridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.conf | 6 ++++++ .../GridStat_fcstHREFmean_obsStgIV_Gempak.conf | 2 -- .../GridStat_fcstHREFmean_obsStgIV_NetCDF.conf | 2 -- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf b/parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf index 6001aae440..f4997cd44d 100644 --- a/parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf +++ b/parm/use_cases/model_applications/climate/GridStat_fcstCESM_obsGFS_ConusTemp.conf @@ -80,9 +80,12 @@ OBS_IS_PROB = false # Output prefix set in grid_stat config file GRID_STAT_OUTPUT_PREFIX={MODEL}_{CURRENT_OBS_NAME}_vs_{OBTYPE} +GRID_STAT_OUTPUT_FLAG_CTC = STAT +GRID_STAT_OUTPUT_FLAG_CTS = STAT GRID_STAT_OUTPUT_FLAG_CNT = STAT GRID_STAT_OUTPUT_FLAG_SL1L2 = STAT + GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE diff --git a/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.conf b/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.conf index 3d3aa4085e..c066d23743 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstHRRR_obsPracPerfect_SurrogateSevereProb.conf @@ -88,6 +88,12 @@ GRID_STAT_OUTPUT_FLAG_PSTD = BOTH GRID_STAT_OUTPUT_FLAG_PJC = BOTH GRID_STAT_OUTPUT_FLAG_PRC = BOTH +GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE +GRID_STAT_NC_PAIRS_FLAG_RAW = FALSE +GRID_STAT_NC_PAIRS_FLAG_DIFF = FALSE +GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE +GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK = FALSE + [dir] # input and output data directories for each application in PROCESS_LIST diff --git a/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.conf b/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.conf index f42ac929a1..eaec762c96 100644 --- a/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.conf +++ b/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_Gempak.conf @@ -73,8 +73,6 @@ GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 GRID_STAT_OUTPUT_FLAG_CTC = STAT GRID_STAT_OUTPUT_FLAG_CTS = STAT -GRID_STAT_OUTPUT_FLAG_ECLV = BOTH -GRID_STAT_OUTPUT_FLAG_GRAD = BOTH GRID_STAT_OUTPUT_FLAG_DMAP = STAT GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE diff --git a/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.conf b/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.conf index 9b8d50df74..42dc08c726 100644 --- a/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.conf +++ b/parm/use_cases/model_applications/precipitation/GridStat_fcstHREFmean_obsStgIV_NetCDF.conf @@ -80,8 +80,6 @@ GRID_STAT_NEIGHBORHOOD_COV_THRESH = >=0.5 GRID_STAT_OUTPUT_FLAG_CTC = STAT GRID_STAT_OUTPUT_FLAG_CTS = STAT -GRID_STAT_OUTPUT_FLAG_ECLV = BOTH -GRID_STAT_OUTPUT_FLAG_GRAD = BOTH GRID_STAT_OUTPUT_FLAG_DMAP = STAT GRID_STAT_NC_PAIRS_FLAG_LATLON = FALSE From 7dfcb72ffb8e60de25408d9e232873572a73df66 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 13:10:29 -0600 Subject: [PATCH 19/43] Revert "fixed typo in value for DMAP" This reverts commit 4ca0bb6c824ab511dcc9dcf43876d5e0c2e67c2c. --- .../GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf b/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf index aab5494da8..98e0430461 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf @@ -40,7 +40,7 @@ GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped GRID_STAT_OUTPUT_PREFIX = FV3_core_{instance} -GRID_STAT_OUTPUT_FLAG_DMAP = BOTH +GRID_STAT_OUTPUT_FLAG_DMAP = DMAP GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = TRUE From 24f17539d6cc6653689883342b7d1170d78932c2 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 13:20:51 -0600 Subject: [PATCH 20/43] removed script that is no longer used --- ci/jobs/run_use_cases.py | 85 ---------------------------------------- metplus/util/met_util.py | 2 +- 2 files changed, 1 insertion(+), 86 deletions(-) delete mode 100755 ci/jobs/run_use_cases.py diff --git a/ci/jobs/run_use_cases.py b/ci/jobs/run_use_cases.py deleted file mode 100755 index c4d4acb69d..0000000000 --- a/ci/jobs/run_use_cases.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import os -import re -import shlex -import subprocess -from os.path import dirname - -import get_data_volumes -import download_gempaktocf -import print_log_errors - -import get_use_case_commands -from metplus.util.met_util import expand_int_string_to_list - -def main(categories, subset_list): - - categories_list = categories.split(',') - - OWNER_BUILD_DIR = os.path.dirname(os.environ['GITHUB_WORKSPACE']) - - # get data volumes - print(f"calling get_data_volumes.main({categories_list})") - volumes_from = get_data_volumes.main(categories_list) - - # obtain GempakToCF.jar for cases that read GEMPAK data - input_data_directory = os.path.join(OWNER_BUILD_DIR, - 'input') - download_gempaktocf.run(input_data_directory) - - # becomes False if any use case fails - isOK = True - - # run use cases - work_dir = os.path.join(os.environ.get('DOCKER_WORK_DIR'), - 'METplus') - all_commands = get_use_case_commands.main(categories_list, - subset_list, - work_dir=work_dir) - for command, requirements in all_commands: - travis_build_dir = os.environ['GITHUB_WORKSPACE'] - if requirements: - reqs = f"{';'.join(requirements)};" - else: - reqs = '' - cmd = (f'{travis_build_dir}/ci/jobs/docker_run_metplus.sh' - f' "{reqs}{command}" "{volumes_from}"') - print(cmd) - try: - subprocess.run(shlex.split(cmd), check=True) - except subprocess.CalledProcessError as err: - print(f"ERROR: Command failed: {cmd} -- {err}") - isOK = False - output_dir = os.path.join(OWNER_BUILD_DIR, - 'output') - replacement_dir = os.path.join(os.environ['DOCKER_DATA_DIR'], - 'output') - print_log_errors.run(output_dir, - replacement_dir) - - # if any tests failed, exit 1, otherwise exit 0 - if not isOK: - sys.exit(1) - -def handle_command_line_args(): - # read command line arguments to determine which use cases to run - if len(sys.argv) < 2: - print("No use cases specified") - sys.exit(1) - - # split up categories by & or , - categories = sys.argv[1] - - # get subset values if specified - if len(sys.argv) > 2: - subset_list = expand_int_string_to_list(sys.argv[2]) - else: - subset_list = None - - return categories, subset_list - -if __name__ == "__main__": - categories, subset_list = handle_command_line_args() - main(categories, subset_list) diff --git a/metplus/util/met_util.py b/metplus/util/met_util.py index bab559b872..2d8dd27341 100644 --- a/metplus/util/met_util.py +++ b/metplus/util/met_util.py @@ -2610,7 +2610,7 @@ def expand_int_string_to_list(int_string): """! Expand string into a list of integer values. Items are separated by commas. Items that are formatted X-Y will be expanded into each number from X to Y inclusive. If the string ends with +, then add a str '+' - to the end of the list. Used in ci/jobs/run_use_cases.py + to the end of the list. Used in ci/jobs/get_use_case_commands.py @param int_string String containing a comma-separated list of integers @returns List of integers and potentially '+' as the last item From a10ac9140ee1143b52b841997a59670c981ba727 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 13:21:16 -0600 Subject: [PATCH 21/43] added info for checking default values in MET configs against share/met/config defaults --- metplus/util/doc_util.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/metplus/util/doc_util.py b/metplus/util/doc_util.py index 4c53b6ece6..2900d1961c 100755 --- a/metplus/util/doc_util.py +++ b/metplus/util/doc_util.py @@ -129,6 +129,11 @@ def print_doc_text(tool_name, met_var, dict_items): print(f'#{mp_config} =') print('\n\n==================================================\n') + print(f"In the parm/met_config/{wrapper_camel}Config_wrapped file, " + f"compare the default values set for {met_var} to the version" + f" in share/met/config/{wrapper_camel}Config_default. If " + "they do differ, make sure to add variables to the use case " + "config files so that they produce the same output.\n\n") print(f"In the parm/met_config/{wrapper_camel}Config_wrapped file, " "replace:\n\n") print(f"{met_var} = ...\n\n with:\n\n//{met_var} =" From 247f2deb3e003512cfc68dc69af1fb0730f954d1 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 13:24:02 -0600 Subject: [PATCH 22/43] rename script name to be more clear, ci-run-all-cases --- ci/actions/run_tests/entrypoint.sh | 2 +- ci/jobs/{run_use_cases_docker.py => run_use_cases.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename ci/jobs/{run_use_cases_docker.py => run_use_cases.py} (100%) diff --git a/ci/actions/run_tests/entrypoint.sh b/ci/actions/run_tests/entrypoint.sh index 4c6fb23be0..a854f82dfe 100644 --- a/ci/actions/run_tests/entrypoint.sh +++ b/ci/actions/run_tests/entrypoint.sh @@ -51,7 +51,7 @@ fi pip_command="pip3 install Pillow" # build command to run -command="./ci/jobs/run_use_cases_docker.py ${CATEGORIES} ${SUBSETLIST}" +command="./ci/jobs/run_use_cases.py ${CATEGORIES} ${SUBSETLIST}" # add input volumes to run command # keep track of --volumes-from arguments to docker run command diff --git a/ci/jobs/run_use_cases_docker.py b/ci/jobs/run_use_cases.py similarity index 100% rename from ci/jobs/run_use_cases_docker.py rename to ci/jobs/run_use_cases.py From c34926fd3b85631e8836856789af07fce706fcf9 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 14:14:31 -0600 Subject: [PATCH 23/43] removed unused scripts --- ci/jobs/download_gempaktocf.py | 38 ------------ ci/jobs/get_artifacts.py | 46 --------------- ci/jobs/gha_get_current_branch.sh | 1 - ci/jobs/gha_get_dockerhub_tag.sh | 2 - ci/jobs/print_log_errors.py | 98 ------------------------------- 5 files changed, 185 deletions(-) delete mode 100755 ci/jobs/download_gempaktocf.py delete mode 100755 ci/jobs/get_artifacts.py delete mode 100755 ci/jobs/gha_get_current_branch.sh delete mode 100755 ci/jobs/gha_get_dockerhub_tag.sh delete mode 100755 ci/jobs/print_log_errors.py diff --git a/ci/jobs/download_gempaktocf.py b/ci/jobs/download_gempaktocf.py deleted file mode 100755 index 3201f3cf93..0000000000 --- a/ci/jobs/download_gempaktocf.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python3 - -"""! Downlaods GempakToCF.jar that is used to convert GEMPAK data to NetCDF""" - -import os -import shlex -import subprocess - -GEMPAK_TO_CF_URL = ('https://dtcenter.org/sites/default/files/community-code/' - 'metplus/utilities/GempakToCF.jar') - -def run(input_data_directory): - """! Downloads GempakToCF.jar from website into input_data_directory - - @param input_data_directory destination to download file - @returns True if successful, False if an error occurred - """ - if not os.path.exists(input_data_directory): - print(f"Creating directory: {input_data_directory}") - os.makedirs(input_data_directory) - - print(f"Downloading {GEMPAK_TO_CF_URL} into {input_data_directory}") - - cmd = f"curl -L -O {GEMPAK_TO_CF_URL}" - try: - subprocess.run(shlex.split(cmd), - check=True, - cwd=input_data_directory) - except subprocess.CalledProcessError as err: - print(f"ERROR: Download failed: {os.path.basename(GEMPAK_TO_CF_URL)}") - return False - - return True - -if __name__ == "__main__": - input_data_directory = os.path.join(os.environ['OWNER_BUILD_DIR'], - 'input') - run(input_data_directory) diff --git a/ci/jobs/get_artifacts.py b/ci/jobs/get_artifacts.py deleted file mode 100755 index 895590429e..0000000000 --- a/ci/jobs/get_artifacts.py +++ /dev/null @@ -1,46 +0,0 @@ -#! /usr/bin/env python3 - -import os -import sys -import subprocess -import shlex - -names = sys.argv[1].split(',') - -DEVELOP_REF_RUN = 552853822 - -ARTIFACT_IDS_DEVELOP = { - 'air_quality_and_comp': 39369666, - 'climate': 39369667, - 'convection_allowing_models_a': 39369668, - 'convection_allowing_models_b': 39369669, - 'cryosphere': 39369670, - 'data_assimilation': 39369671, - 'medium_range_a': 39369672, - 'medium_range_b': 39369673, - 'medium_range_c': 39369674, - 'met_tool_wrapper': 39369675, - 'precipitation': 39369676, -} - -for name in names: - artifact_id = ARTIFACT_IDS_DEVELOP[name] - my_secret_api_token = os.environ.get('MY_SECRET_API_TOKEN') - artifact_url = f"https://{my_secret_api_token}@api.github.com/repos/dtcenter/metplus/actions/artifacts/{artifact_id}/zip" - - truth_dir = os.path.abspath(os.path.join(os.environ.get('GITHUB_WORKSPACE'), - os.pardir, - 'truth')) - - output_file = os.path.join(truth_dir, f'{name}.zip') - - if not os.path.exists(truth_dir): - print(f"Creating directory: {truth_dir}") - os.makedirs(truth_dir) - - cmd = f'curl -L -o {output_file} {artifact_url}' - print(cmd) - ret = subprocess.run(shlex.split(cmd)) - - cmd = f'unzip {output_file} -d {truth_dir}/' - ret = subprocess.run(shlex.split(cmd)) diff --git a/ci/jobs/gha_get_current_branch.sh b/ci/jobs/gha_get_current_branch.sh deleted file mode 100755 index e2af4e54a3..0000000000 --- a/ci/jobs/gha_get_current_branch.sh +++ /dev/null @@ -1 +0,0 @@ -echo ${GITHUB_REF#refs/heads/} diff --git a/ci/jobs/gha_get_dockerhub_tag.sh b/ci/jobs/gha_get_dockerhub_tag.sh deleted file mode 100755 index d1a78a4bc2..0000000000 --- a/ci/jobs/gha_get_dockerhub_tag.sh +++ /dev/null @@ -1,2 +0,0 @@ -current_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -echo dtcenter/metplus-dev:`$current_dir/gha_get_current_branch.sh` diff --git a/ci/jobs/print_log_errors.py b/ci/jobs/print_log_errors.py deleted file mode 100755 index aa1ed68559..0000000000 --- a/ci/jobs/print_log_errors.py +++ /dev/null @@ -1,98 +0,0 @@ -#! /usr/bin/env python3 - -import sys -import os -import glob - -# strings to search for to find errors -error_str = 'ERROR' -command_start_str = 'COPYABLE ENVIRONMENT FOR NEXT COMMAND' -check_logfile_string = "Check the logfile for more information on why it failed" - -def run(output_dir, replacement_dir=None): - """! Search for logs under output_dir that have MET errors and print the - commands that have errors to the screen. Assumes logs are either in - {output_dir}/logs or {output_dir}//logs where is - any subdirectory under the output directory - @param output_dir directory to search for logs - @param replacement_dir directory to replace for output_dir to find - files. This is needed when running through Travis because the - directory in Travis differs from the directory listed in the log - files because the are run in the Docker container. - """ - # pytest logs directory is in top level of output_dir - log_topdir_glob = os.path.join(output_dir, - 'logs', - '*') - - # use case logs directory are in each subdirectory in output_dir - log_subdir_glob = os.path.join(output_dir, - '*', - 'logs', - '*') - logs_with_errors = set() - - # get list of log files that have MET errors in them using check_logfile_string - all_log_files = glob.glob(log_topdir_glob) + glob.glob(log_subdir_glob) - for log_file in all_log_files: - - with open(log_file, 'r') as file_handle: - lines = file_handle.readlines() - - for line in lines: - if check_logfile_string in line: - error_log = line.split(':')[-1].strip() - logs_with_errors.add(error_log) - - # find error lines and get text for command that contains the error - for log_file in logs_with_errors: - try: - if replacement_dir: - local_log_file = log_file.replace(replacement_dir, output_dir) - - with open(local_log_file, 'r') as file_handle: - lines = file_handle.readlines() - except OSError: - print(f"ERROR: Could not open {local_log_file}") - continue - - error_indices = [] - command_start_indices = [] - for index, line in enumerate(lines): - if line.startswith(error_str): - error_indices.append(index) - if command_start_str in line: - command_start_indices.append(index) - - if not error_indices: - continue - - lines_to_print = set() - for index, command_start_2 in enumerate(command_start_indices): - if index == 0: - continue - - command_start_1 = command_start_indices[index-1] - # check if any error lines are found between command start lines - for error_index in error_indices: - if error_index > command_start_1 and error_index < command_start_2: - lines_to_print.add((command_start_1, command_start_2)) - - for start_index, end_index in lines_to_print: - print(f"\nPrinting lines {start_index} to {end_index} " - f"from {local_log_file}") - for line in lines[start_index:end_index]: - print(line.strip()) - -if __name__ == "__main__": - if len(sys.argv) < 2: - print("ERROR: Must supply directory to search as argument") - sys.exit(1) - elif len(sys.argv) > 2: - replacement_dir = sys.argv[2] - else: - replacement_dir = None - - output_dir = sys.argv[1] - run(output_dir, replacement_dir) - From 92cdfa17f7ebf50611a8f418a4054ace46229f84 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 14:15:12 -0600 Subject: [PATCH 24/43] added unsupported file types for diff logic so the output can be obtained and compared by hand until scripts can be improved to handle the data --- ci/util/diff_util.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ci/util/diff_util.py b/ci/util/diff_util.py index 6d14257e8b..9fd79c283a 100644 --- a/ci/util/diff_util.py +++ b/ci/util/diff_util.py @@ -8,7 +8,6 @@ '.png', '.jpg', '.jpeg', - '.pdf', ] NETCDF_EXTENSIONS = [ @@ -20,6 +19,10 @@ '.zip', ] +UNSUPPORTED_EXTENSIONS = [ + '.pdf', +] + def get_file_type(filepath): _, file_extension = os.path.splitext(filepath) if file_extension in IMAGE_EXTENSIONS: @@ -40,6 +43,9 @@ def get_file_type(filepath): if file_extension in SKIP_EXTENSIONS: return 'skip' + if file_extension in UNSUPPORTED_EXTENSIONS: + return f'unsupported{file_extension}' + return 'unknown' def compare_dir(dir_a, dir_b, debug=False): @@ -128,6 +134,10 @@ def compare_files(filepath_a, filepath_b, debug=False, dir_a=None, dir_b=None): print(f'Skipping') return None + if file_type.startswith('unsupported'): + print(f"Unsupported file type encountered: {file_type.split('.')[1]}") + return (filepath_a, filepath_b, file_type) + if file_type == 'netcdf': print("Comparing NetCDF") if not nc_is_equal(filepath_a, filepath_b): From f3b632a3dbba4df9b50b0a2b59873c4f040d4513 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 14:15:47 -0600 Subject: [PATCH 25/43] added comments to scripts to explain what they do and what calls them in automation --- ci/jobs/create_output_data_volumes.sh | 4 ++++ ci/jobs/diff_output.py | 3 +++ ci/jobs/docker_setup.sh | 8 ++++++++ ci/jobs/docker_update_data_volumes.py | 5 +++++ ci/jobs/docker_utils.py | 5 +++++ ci/jobs/get_artifact_name.sh | 4 ++++ ci/jobs/get_data_volumes.py | 5 +++++ ci/jobs/get_use_case_commands.py | 4 ++++ ci/jobs/print_branch_name.py | 4 ++++ ci/jobs/print_python_version.py | 3 +++ ci/jobs/run_use_cases.py | 6 ++++++ ci/jobs/set_job_controls.sh | 4 ++++ 12 files changed, 55 insertions(+) diff --git a/ci/jobs/create_output_data_volumes.sh b/ci/jobs/create_output_data_volumes.sh index 7a65e68fc8..ca41ca8291 100755 --- a/ci/jobs/create_output_data_volumes.sh +++ b/ci/jobs/create_output_data_volumes.sh @@ -1,5 +1,9 @@ #! /bin/bash +# Run by GitHub Actions (in .github/workflows/main.yml) to create +# Docker data volumes from output data to create a "truth" +# data set to use in difference tests. + if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then echo This is a pull request, so skip this setp exit 0 diff --git a/ci/jobs/diff_output.py b/ci/jobs/diff_output.py index 353523a37f..969e474497 100755 --- a/ci/jobs/diff_output.py +++ b/ci/jobs/diff_output.py @@ -1,5 +1,8 @@ #! /usr/bin/env python3 +# Run by GitHub Actions (in ci/jobs/run_use_cases.py) to +# trigger difference tests. + import sys import os diff --git a/ci/jobs/docker_setup.sh b/ci/jobs/docker_setup.sh index 9d890b23d7..93719ce1e2 100755 --- a/ci/jobs/docker_setup.sh +++ b/ci/jobs/docker_setup.sh @@ -1,5 +1,13 @@ #! /bin/bash +# Run by GitHub Actions (in .github/workflows/main.yml) to build +# METplus Docker image and put it up to DockerHub so it can be +# used by the use case tests. +# If GitHub Actions run is triggered by a fork that does not have +# permissions to push Docker images to DockerHub, the script is +# is also called (in ci/actions/run_tests/entrypoint.sh) to +# build the Docker image to use for each use case test group + branch_name=`${GITHUB_WORKSPACE}/ci/jobs/print_branch_name.py` if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then branch_name=${branch_name}-pull_request diff --git a/ci/jobs/docker_update_data_volumes.py b/ci/jobs/docker_update_data_volumes.py index 5d7e6748c2..9098f56684 100755 --- a/ci/jobs/docker_update_data_volumes.py +++ b/ci/jobs/docker_update_data_volumes.py @@ -1,5 +1,10 @@ #! /usr/bin/env python3 +# Run by GitHub Actions (in .github/workflows/main.yml) check DTCenter web +# server for any input data tarfiles that have been updated and need to be +# regenerated as Docker data volumes to be used in use case tests. +# Push new/updated data volumes up to DockerHub + import sys import os import shlex diff --git a/ci/jobs/docker_utils.py b/ci/jobs/docker_utils.py index b094bc8c70..1bc5d6552c 100644 --- a/ci/jobs/docker_utils.py +++ b/ci/jobs/docker_utils.py @@ -1,6 +1,11 @@ import os import re +# Utilities used by various CI jobs. Functionality includes: +# - Check if Docker data volumes need to be updated. +# - Get appropriate branch name to use to obtain/create Docker +# images. This is needed for pull request runs. + # repository used for storing input data for development branches DOCKERHUB_DATA_REPO = 'dtcenter/metplus-data-dev' diff --git a/ci/jobs/get_artifact_name.sh b/ci/jobs/get_artifact_name.sh index b56a8b5c4d..e2446b4d91 100755 --- a/ci/jobs/get_artifact_name.sh +++ b/ci/jobs/get_artifact_name.sh @@ -1,5 +1,9 @@ #! /bin/bash +# Run by GitHub Actions (in .github/workflows/main.yml and +# ci/actions/run_tests/entrypoint.sh) to get properly +# formatted artifact name for use case output + artifact_name=$1 # strip of :NEW if found at end of name if [ ${artifact_name: -4} == ":NEW" ]; then diff --git a/ci/jobs/get_data_volumes.py b/ci/jobs/get_data_volumes.py index d1e96601ef..86ea3c6ec7 100755 --- a/ci/jobs/get_data_volumes.py +++ b/ci/jobs/get_data_volumes.py @@ -1,5 +1,10 @@ #! /usr/bin/env python3 +# Run by GitHub Actions (in ci/actions/run_tests/entrypoint.sh) +# to obtain Docker data volumes for input and output data, create +# an alias name for the volumes, and generate --volumes-from arguments +# that are added to the Docker run command to make data available + import sys import os import subprocess diff --git a/ci/jobs/get_use_case_commands.py b/ci/jobs/get_use_case_commands.py index 0eceaca127..610a1f9cd5 100755 --- a/ci/jobs/get_use_case_commands.py +++ b/ci/jobs/get_use_case_commands.py @@ -1,5 +1,9 @@ #! /usr/bin/env python3 +# Script to obtain commands needed to run use case groups including +# scripts or pip commands to obtain external Python dependencies +# Run by GitHub Actions (in ci/jobs/run_use_cases.py) to run use case tests + import sys import os diff --git a/ci/jobs/print_branch_name.py b/ci/jobs/print_branch_name.py index b92aacbe9f..b3bfcd299a 100755 --- a/ci/jobs/print_branch_name.py +++ b/ci/jobs/print_branch_name.py @@ -1,5 +1,9 @@ #! /usr/bin/env python3 +# Script to easily get branch name from docker_utils function +# Run by GitHub Actions (in ci/actions/run_tests/entrypoint.sh, +# ci/jobs/create_output_data_volumes.sh, and ci/jobs/docker_setup.sh) + from docker_utils import get_branch_name print(get_branch_name()) diff --git a/ci/jobs/print_python_version.py b/ci/jobs/print_python_version.py index 7de32fe4f0..6c7abebf58 100755 --- a/ci/jobs/print_python_version.py +++ b/ci/jobs/print_python_version.py @@ -1,5 +1,8 @@ #! /usr/bin/env python3 +# Script to easily obtain minimum python version requirement +# Used in GitHub Actions (in ci/jobs/get_miniconda.sh) + import sys import os diff --git a/ci/jobs/run_use_cases.py b/ci/jobs/run_use_cases.py index 05b395dbab..1ca795c1a3 100755 --- a/ci/jobs/run_use_cases.py +++ b/ci/jobs/run_use_cases.py @@ -1,5 +1,11 @@ #! /usr/bin/env python3 +# Used in GitHub Actions (in ci/actions/run_tests/entrypoint.sh) +# to obtain and run commands to run use cases from group, +# execute difference tests if requested, copy error logs and/or +# files that reported differences into directory to make +# them available in GitHub Actions artifacts for easy review + import os import sys import subprocess diff --git a/ci/jobs/set_job_controls.sh b/ci/jobs/set_job_controls.sh index 9dd6e14ae9..cea137273a 100755 --- a/ci/jobs/set_job_controls.sh +++ b/ci/jobs/set_job_controls.sh @@ -1,5 +1,9 @@ #! /bin/bash +# Run by GitHub Actions (in .github/workflows/main.yml) to parse +# info from GitHub event and commit message from last commit before +# a push to determine which jobs to run and which to skip. + # set default status for jobs run_docs=true run_get_image=true From 1aefe3566a0b846d7b5aba93dd486c81d460c5ee Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 14:22:10 -0600 Subject: [PATCH 26/43] moved scripts to obtain external python requirements into a directory to avoid cluttering up job directory, ci-run-diff --- ci/jobs/get_use_case_commands.py | 1 + ci/jobs/print_python_version.py | 2 +- ci/jobs/{ => python_requirements}/get_cartopy.sh | 0 ci/jobs/{ => python_requirements}/get_metcalcpy.sh | 2 +- ci/jobs/{ => python_requirements}/get_metplotpy.sh | 2 +- ci/jobs/{ => python_requirements}/get_miniconda.sh | 2 +- ci/jobs/{ => python_requirements}/get_pygrib.sh | 1 - ci/jobs/{ => python_requirements}/get_xesmf.sh | 0 8 files changed, 5 insertions(+), 5 deletions(-) rename ci/jobs/{ => python_requirements}/get_cartopy.sh (100%) rename ci/jobs/{ => python_requirements}/get_metcalcpy.sh (91%) rename ci/jobs/{ => python_requirements}/get_metplotpy.sh (91%) rename ci/jobs/{ => python_requirements}/get_miniconda.sh (93%) rename ci/jobs/{ => python_requirements}/get_pygrib.sh (83%) rename ci/jobs/{ => python_requirements}/get_xesmf.sh (100%) diff --git a/ci/jobs/get_use_case_commands.py b/ci/jobs/get_use_case_commands.py index 610a1f9cd5..597e8e80a2 100755 --- a/ci/jobs/get_use_case_commands.py +++ b/ci/jobs/get_use_case_commands.py @@ -23,6 +23,7 @@ def handle_requirements(requirements, work_dir): script_path = os.path.join(work_dir, 'ci', 'jobs', + 'python_requirements', f'get_{requirement.lower()}.sh') print(f"Looking for script: {script_path}") if os.path.exists(script_path): diff --git a/ci/jobs/print_python_version.py b/ci/jobs/print_python_version.py index 6c7abebf58..5b6e382354 100755 --- a/ci/jobs/print_python_version.py +++ b/ci/jobs/print_python_version.py @@ -1,7 +1,7 @@ #! /usr/bin/env python3 # Script to easily obtain minimum python version requirement -# Used in GitHub Actions (in ci/jobs/get_miniconda.sh) +# Used in GitHub Actions (in ci/jobs/python_requirements/get_miniconda.sh) import sys import os diff --git a/ci/jobs/get_cartopy.sh b/ci/jobs/python_requirements/get_cartopy.sh similarity index 100% rename from ci/jobs/get_cartopy.sh rename to ci/jobs/python_requirements/get_cartopy.sh diff --git a/ci/jobs/get_metcalcpy.sh b/ci/jobs/python_requirements/get_metcalcpy.sh similarity index 91% rename from ci/jobs/get_metcalcpy.sh rename to ci/jobs/python_requirements/get_metcalcpy.sh index 192a3b638e..86837d0aef 100755 --- a/ci/jobs/get_metcalcpy.sh +++ b/ci/jobs/python_requirements/get_metcalcpy.sh @@ -5,7 +5,7 @@ pip3 install scipy pip3 install pingouin basedir=$(dirname "$0") -work_dir=$basedir/../.. +work_dir=$basedir/../../.. # run manage externals to obtain METcalcpy ${work_dir}/manage_externals/checkout_externals -e ${work_dir}/ci/parm/Externals_metcalcpy.cfg diff --git a/ci/jobs/get_metplotpy.sh b/ci/jobs/python_requirements/get_metplotpy.sh similarity index 91% rename from ci/jobs/get_metplotpy.sh rename to ci/jobs/python_requirements/get_metplotpy.sh index cb79c9faea..1fc1f4b64c 100755 --- a/ci/jobs/get_metplotpy.sh +++ b/ci/jobs/python_requirements/get_metplotpy.sh @@ -5,7 +5,7 @@ pip3 install scipy pip3 install cmocean basedir=$(dirname "$0") -work_dir=$basedir/../.. +work_dir=$basedir/../../.. # run manage externals to obtain METcalcpy ${work_dir}/manage_externals/checkout_externals -e ${work_dir}/ci/parm/Externals_metplotpy.cfg diff --git a/ci/jobs/get_miniconda.sh b/ci/jobs/python_requirements/get_miniconda.sh similarity index 93% rename from ci/jobs/get_miniconda.sh rename to ci/jobs/python_requirements/get_miniconda.sh index 0f6dc4a10e..ea644774fd 100755 --- a/ci/jobs/get_miniconda.sh +++ b/ci/jobs/python_requirements/get_miniconda.sh @@ -2,7 +2,7 @@ script_dir=$(dirname "$0") -python_version=`${script_dir}/print_python_version.py` +python_version=`${script_dir}/../print_python_version.py` # these are used to obtain version of MiniConda3 # the version determines the default version of Python diff --git a/ci/jobs/get_pygrib.sh b/ci/jobs/python_requirements/get_pygrib.sh similarity index 83% rename from ci/jobs/get_pygrib.sh rename to ci/jobs/python_requirements/get_pygrib.sh index 18d28318f0..6f1b63aa05 100755 --- a/ci/jobs/get_pygrib.sh +++ b/ci/jobs/python_requirements/get_pygrib.sh @@ -1,7 +1,6 @@ #!/bin/bash #shell script to install pygrib with dependencies -# todd arbetter: arbetter@ucar.edu yum -y install eccodes-devel pip3 install numpy diff --git a/ci/jobs/get_xesmf.sh b/ci/jobs/python_requirements/get_xesmf.sh similarity index 100% rename from ci/jobs/get_xesmf.sh rename to ci/jobs/python_requirements/get_xesmf.sh From 1fbfffba1e85989d55b717203f8651cc6ca3bbf2 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 14:24:14 -0600 Subject: [PATCH 27/43] change typo back to correct version since I confirmed the error logs artifact only contains the log directories that have errors --- .../GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf b/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf index 98e0430461..aab5494da8 100644 --- a/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf +++ b/parm/use_cases/model_applications/convection_allowing_models/GridStat_fcstFV3_obsGOES_BrightnessTempDmap.conf @@ -40,7 +40,7 @@ GRID_STAT_CONFIG_FILE = {PARM_BASE}/met_config/GridStatConfig_wrapped GRID_STAT_OUTPUT_PREFIX = FV3_core_{instance} -GRID_STAT_OUTPUT_FLAG_DMAP = DMAP +GRID_STAT_OUTPUT_FLAG_DMAP = BOTH GRID_STAT_NC_PAIRS_FLAG_CLIMO = FALSE GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP = TRUE From 667574f37a54d1b4ae903684a3061ed98d521e62 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 16:15:39 -0600 Subject: [PATCH 28/43] updated doc util script to help generate unit tests, added tests for new variables --- .../grid_stat/test_grid_stat_wrapper.py | 166 ++++++++++++++++++ .../point_stat/test_point_stat_wrapper.py | 16 ++ metplus/util/doc_util.py | 42 +++++ 3 files changed, 224 insertions(+) diff --git a/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py b/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py index d49a2fc858..9fd939fdcd 100644 --- a/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py +++ b/internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py @@ -120,6 +120,172 @@ ({'GRID_STAT_OUTPUT_PREFIX': 'my_output_prefix'}, {'METPLUS_OUTPUT_PREFIX': 'output_prefix = "my_output_prefix";'}), + + ({'GRID_STAT_OUTPUT_FLAG_FHO': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {fho = STAT;}'}), + + ({'GRID_STAT_OUTPUT_FLAG_CTC': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {ctc = STAT;}'}), + + ({'GRID_STAT_OUTPUT_FLAG_CTS': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {cts = STAT;}'}), + + ({'GRID_STAT_OUTPUT_FLAG_MCTC': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {mctc = STAT;}'}), + + ({'GRID_STAT_OUTPUT_FLAG_MCTS': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {mcts = STAT;}'}), + + ({'GRID_STAT_OUTPUT_FLAG_CNT': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {cnt = STAT;}'}), + + ({'GRID_STAT_OUTPUT_FLAG_SL1L2': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {sl1l2 = STAT;}'}), + + ({'GRID_STAT_OUTPUT_FLAG_SAL1L2': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {sal1l2 = STAT;}'}), + + ({'GRID_STAT_OUTPUT_FLAG_VL1L2': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {vl1l2 = STAT;}'}), + + ({'GRID_STAT_OUTPUT_FLAG_VAL1L2': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {val1l2 = STAT;}'}), + + ({'GRID_STAT_OUTPUT_FLAG_VCNT': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {vcnt = STAT;}'}), + + ({'GRID_STAT_OUTPUT_FLAG_PCT': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {pct = STAT;}'}), + + ({'GRID_STAT_OUTPUT_FLAG_PSTD': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {pstd = STAT;}'}), + + ({'GRID_STAT_OUTPUT_FLAG_PJC': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {pjc = STAT;}'}), + + ({'GRID_STAT_OUTPUT_FLAG_PRC': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {prc = STAT;}'}), + + ({'GRID_STAT_OUTPUT_FLAG_ECLV': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {eclv = STAT;}'}), + + ({'GRID_STAT_OUTPUT_FLAG_NBRCTC': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {nbrctc = STAT;}'}), + + ({'GRID_STAT_OUTPUT_FLAG_NBRCTS': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {nbrcts = STAT;}'}), + + ({'GRID_STAT_OUTPUT_FLAG_NBRCNT': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {nbrcnt = STAT;}'}), + + ({'GRID_STAT_OUTPUT_FLAG_GRAD': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {grad = STAT;}'}), + + ({'GRID_STAT_OUTPUT_FLAG_DMAP': 'STAT', }, + {'METPLUS_OUTPUT_FLAG_DICT': 'output_flag = {dmap = STAT;}'}), + + ({ + 'GRID_STAT_OUTPUT_FLAG_FHO': 'STAT', + 'GRID_STAT_OUTPUT_FLAG_CTC': 'STAT', + 'GRID_STAT_OUTPUT_FLAG_CTS': 'STAT', + 'GRID_STAT_OUTPUT_FLAG_MCTC': 'STAT', + 'GRID_STAT_OUTPUT_FLAG_MCTS': 'STAT', + 'GRID_STAT_OUTPUT_FLAG_CNT': 'STAT', + 'GRID_STAT_OUTPUT_FLAG_SL1L2': 'STAT', + 'GRID_STAT_OUTPUT_FLAG_SAL1L2': 'STAT', + 'GRID_STAT_OUTPUT_FLAG_VL1L2': 'STAT', + 'GRID_STAT_OUTPUT_FLAG_VAL1L2': 'STAT', + 'GRID_STAT_OUTPUT_FLAG_VCNT': 'STAT', + 'GRID_STAT_OUTPUT_FLAG_PCT': 'STAT', + 'GRID_STAT_OUTPUT_FLAG_PSTD': 'STAT', + 'GRID_STAT_OUTPUT_FLAG_PJC': 'STAT', + 'GRID_STAT_OUTPUT_FLAG_PRC': 'STAT', + 'GRID_STAT_OUTPUT_FLAG_ECLV': 'STAT', + 'GRID_STAT_OUTPUT_FLAG_NBRCTC': 'STAT', + 'GRID_STAT_OUTPUT_FLAG_NBRCTS': 'STAT', + 'GRID_STAT_OUTPUT_FLAG_NBRCNT': 'STAT', + 'GRID_STAT_OUTPUT_FLAG_GRAD': 'STAT', + 'GRID_STAT_OUTPUT_FLAG_DMAP': 'STAT', + }, + { + 'METPLUS_OUTPUT_FLAG_DICT': ( + 'output_flag = {fho = STAT;ctc = STAT;cts = STAT;' + 'mctc = STAT;mcts = STAT;cnt = STAT;sl1l2 = STAT;' + 'sal1l2 = STAT;vl1l2 = STAT;val1l2 = STAT;' + 'vcnt = STAT;pct = STAT;pstd = STAT;pjc = STAT;' + 'prc = STAT;eclv = STAT;nbrctc = STAT;nbrcts = STAT;' + 'nbrcnt = STAT;grad = STAT;dmap = STAT;}')}), + + ({'GRID_STAT_NC_PAIRS_FLAG_LATLON': 'TRUE', }, + {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {latlon = TRUE;}'}), + + ({'GRID_STAT_NC_PAIRS_FLAG_RAW': 'TRUE', }, + {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {raw = TRUE;}'}), + + ({'GRID_STAT_NC_PAIRS_FLAG_DIFF': 'TRUE', }, + {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {diff = TRUE;}'}), + + ({'GRID_STAT_NC_PAIRS_FLAG_CLIMO': 'TRUE', }, + {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {climo = TRUE;}'}), + + ({'GRID_STAT_NC_PAIRS_FLAG_CLIMO_CDP': 'TRUE', }, + { + 'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {climo_cdp = TRUE;}'}), + + ({'GRID_STAT_NC_PAIRS_FLAG_WEIGHT': 'TRUE', }, + {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {weight = TRUE;}'}), + + ({'GRID_STAT_NC_PAIRS_FLAG_NBRHD': 'TRUE', }, + {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {nbrhd = TRUE;}'}), + + ({'GRID_STAT_NC_PAIRS_FLAG_FOURIER': 'TRUE', }, + {'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {fourier = TRUE;}'}), + + ({'GRID_STAT_NC_PAIRS_FLAG_GRADIENT': 'TRUE', }, + { + 'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {gradient = TRUE;}'}), + + ({'GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP': 'TRUE', }, + { + 'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {distance_map = TRUE;}'}), + + ({'GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK': 'TRUE', }, + { + 'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {apply_mask = TRUE;}'}), + + ({ + 'GRID_STAT_NC_PAIRS_FLAG_LATLON': 'TRUE', + 'GRID_STAT_NC_PAIRS_FLAG_RAW': 'TRUE', + 'GRID_STAT_NC_PAIRS_FLAG_DIFF': 'TRUE', + 'GRID_STAT_NC_PAIRS_FLAG_CLIMO': 'TRUE', + 'GRID_STAT_NC_PAIRS_FLAG_CLIMO_CDP': 'TRUE', + 'GRID_STAT_NC_PAIRS_FLAG_WEIGHT': 'TRUE', + 'GRID_STAT_NC_PAIRS_FLAG_NBRHD': 'TRUE', + 'GRID_STAT_NC_PAIRS_FLAG_FOURIER': 'TRUE', + 'GRID_STAT_NC_PAIRS_FLAG_GRADIENT': 'TRUE', + 'GRID_STAT_NC_PAIRS_FLAG_DISTANCE_MAP': 'TRUE', + 'GRID_STAT_NC_PAIRS_FLAG_APPLY_MASK': 'TRUE', + }, + { + 'METPLUS_NC_PAIRS_FLAG_DICT': 'nc_pairs_flag = {latlon = TRUE;raw = TRUE;diff = TRUE;climo = TRUE;climo_cdp = TRUE;weight = TRUE;nbrhd = TRUE;fourier = TRUE;gradient = TRUE;distance_map = TRUE;apply_mask = TRUE;}'}), + + ({'GRID_STAT_CLIMO_CDF_CDF_BINS': '1', }, + {'METPLUS_CLIMO_CDF_DICT': 'climo_cdf = {cdf_bins = 1.0;}'}), + + ({'GRID_STAT_CLIMO_CDF_CENTER_BINS': 'True', }, + {'METPLUS_CLIMO_CDF_DICT': 'climo_cdf = {center_bins = TRUE;}'}), + + ({'GRID_STAT_CLIMO_CDF_WRITE_BINS': 'False', }, + {'METPLUS_CLIMO_CDF_DICT': 'climo_cdf = {write_bins = FALSE;}'}), + + ({ + 'GRID_STAT_CLIMO_CDF_CDF_BINS': '1', + 'GRID_STAT_CLIMO_CDF_CENTER_BINS': 'True', + 'GRID_STAT_CLIMO_CDF_WRITE_BINS': 'False', + }, + { + 'METPLUS_CLIMO_CDF_DICT': 'climo_cdf = {cdf_bins = 1.0;center_bins = TRUE;write_bins = FALSE;}'}), + ] ) def test_grid_stat_single_field(metplus_config, config_overrides, diff --git a/internal_tests/pytests/point_stat/test_point_stat_wrapper.py b/internal_tests/pytests/point_stat/test_point_stat_wrapper.py index 3e60fe832b..38f8200904 100755 --- a/internal_tests/pytests/point_stat/test_point_stat_wrapper.py +++ b/internal_tests/pytests/point_stat/test_point_stat_wrapper.py @@ -135,6 +135,22 @@ def point_stat_wrapper(metplus_config): 'obs_window = {beg = -2700;end = 2700;}', }), + ({'POINT_STAT_CLIMO_CDF_CDF_BINS': '1', }, + {'METPLUS_CLIMO_CDF_DICT': 'climo_cdf = {cdf_bins = 1.0;}'}), + + ({'POINT_STAT_CLIMO_CDF_CENTER_BINS': 'True', }, + {'METPLUS_CLIMO_CDF_DICT': 'climo_cdf = {center_bins = TRUE;}'}), + + ({'POINT_STAT_CLIMO_CDF_WRITE_BINS': 'False', }, + {'METPLUS_CLIMO_CDF_DICT': 'climo_cdf = {write_bins = FALSE;}'}), + + ({ + 'POINT_STAT_CLIMO_CDF_CDF_BINS': '1', + 'POINT_STAT_CLIMO_CDF_CENTER_BINS': 'True', + 'POINT_STAT_CLIMO_CDF_WRITE_BINS': 'False', + }, + { + 'METPLUS_CLIMO_CDF_DICT': 'climo_cdf = {cdf_bins = 1.0;center_bins = TRUE;write_bins = FALSE;}'}), ] ) diff --git a/metplus/util/doc_util.py b/metplus/util/doc_util.py index 2900d1961c..67ad9dbeef 100755 --- a/metplus/util/doc_util.py +++ b/metplus/util/doc_util.py @@ -173,6 +173,48 @@ def print_doc_text(tool_name, met_var, dict_items): f" | *Used by:* {wrapper_camel}") print(f'{glossary_entry}\n') + print('\n==================================================\n') + print(f"In internal_tests/pytests/{tool_name}/" + f"test_{tool_name}_wrapper.py, add the following items to " + "the tests to ensure the new items are set properly. Note: " + "if the tool does not have unit tests to check the handling of " + "MET config variables, you will need to add those tests. See " + "grid_stat/test_grid_stat_wrapper.py for an example. Change " + "VALUE to an appropriate value for the variable.\n\n") + + input_dict_items = [] + output_items = [] + for metplus_config_name, met_config_name in zip(metplus_config_names, met_config_values): + if dict_items: + item_name = met_config_name.split('.')[1] + output_item = f"{item_name} = VALUE;" + else: + output_item = 'VALUE;' + mp_config_dict_item = f"'{metplus_config_name}': 'VALUE'," + input_dict_items.append(mp_config_dict_item) + output_items.append(output_item) + if dict_items: + output_fmt = f"{{{output_item}}}" + else: + output_fmt = output_item + + + test_text = (f" ({{{mp_config_dict_item} }},\n" + f" {{'{env_var_name}': '{met_var} = " + f"{output_fmt}'}}),\n") + print(test_text) + + if dict_items: + all_items_text = " ({\n" + for input_dict_item in input_dict_items: + all_items_text += f" {input_dict_item}\n" + all_items_text += (" },\n" + f" {{'{env_var_name}': '{met_var} = {{") + all_items_text += ''.join(output_items) + all_items_text += "}'})," + print(all_items_text) + + def doc_util_usage(): """! Print usage statement for script """ print(f"{__file__} [ Date: Mon, 29 Mar 2021 16:56:04 -0600 Subject: [PATCH 29/43] added support for diffing PDF files as images, ci-run-diff --- ci/util/diff_util.py | 41 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/ci/util/diff_util.py b/ci/util/diff_util.py index 9fd79c283a..eefa0af3c3 100644 --- a/ci/util/diff_util.py +++ b/ci/util/diff_util.py @@ -19,10 +19,13 @@ '.zip', ] -UNSUPPORTED_EXTENSIONS = [ +PDF_EXTENSIONS = [ '.pdf', ] +UNSUPPORTED_EXTENSIONS = [ +] + def get_file_type(filepath): _, file_extension = os.path.splitext(filepath) if file_extension in IMAGE_EXTENSIONS: @@ -43,6 +46,9 @@ def get_file_type(filepath): if file_extension in SKIP_EXTENSIONS: return 'skip' + if file_extension in PDF_EXTENSIONS: + return 'pdf' + if file_extension in UNSUPPORTED_EXTENSIONS: return f'unsupported{file_extension}' @@ -146,6 +152,14 @@ def compare_files(filepath_a, filepath_b, debug=False, dir_a=None, dir_b=None): print("No differences in NetCDF files") return True + if file_type == 'pdf': + print("Comparing PDF as images") + if not compare_pdf_as_images(filepath_a, filepath_b): + return (filepath_a, filepath_b, 'PDF diff') + + print("No differences in PDF files") + return True + if file_type == 'image': print("Comparing images") if not compare_image_files(filepath_a, filepath_b): @@ -169,23 +183,42 @@ def compare_files(filepath_a, filepath_b, debug=False, dir_a=None, dir_b=None): return True -def compare_image_files(filepath_a, filepath_b): - diff_count = 0 +def compare_pdf_as_images(filepath_a, filepath_b): + try: + from pdf2image import convert_from_path + except ModuleNotFoundError: + print("Cannot compare PDF files without pdf2image Python package") + return False + + all_equal = True + images_a = convert_from_path(filepath_a) + images_b = convert_from_path(filepath_b) + for image_a, image_b in zip(images_a, images_b): + if not compare_images(image_a, image_b): + all_equal = False + + return all_equal +def compare_image_files(filepath_a, filepath_b): image_a = Image.open(filepath_a) image_b = Image.open(filepath_b) + return compare_images(image_a, image_b) + +def compare_images(image_a, image_b): + diff_count = 0 image_diff = ImageChops.difference(image_a, image_b) nx, ny = image_diff.size for x in range(0, int(nx)): for y in range(0, int(ny)): pixel = image_diff.getpixel((x, y)) - if pixel != 0 and pixel != (0, 0, 0, 0): + if pixel != 0 and pixel != (0, 0, 0, 0) and pixel != (0, 0, 0): diff_count += 1 if diff_count: print(f"ERROR: Found {diff_count} differences between images") return False return True + def compare_txt_files(filepath_a, filepath_b, dir_a=None, dir_b=None): with open(filepath_a, 'r') as file_handle: lines_a = file_handle.read().splitlines() From 086ac54f101d82ff364e1ee1507a03c0d888c4a0 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 17:44:55 -0600 Subject: [PATCH 30/43] updated diff logic to convert pdf files to images to compare, added option to save diff image to inspect --- ci/actions/run_tests/entrypoint.sh | 2 +- ci/jobs/run_use_cases.py | 9 ++- ci/util/diff_util.py | 96 +++++++++++++++++++++--------- 3 files changed, 76 insertions(+), 31 deletions(-) diff --git a/ci/actions/run_tests/entrypoint.sh b/ci/actions/run_tests/entrypoint.sh index a854f82dfe..0720217424 100644 --- a/ci/actions/run_tests/entrypoint.sh +++ b/ci/actions/run_tests/entrypoint.sh @@ -48,7 +48,7 @@ fi # install Pillow library needed for diff testing # this will be replaced with better image diffing package used by METplotpy -pip_command="pip3 install Pillow" +pip_command="pip3 install Pillow; pip3 install pdf2image" # build command to run command="./ci/jobs/run_use_cases.py ${CATEGORIES} ${SUBSETLIST}" diff --git a/ci/jobs/run_use_cases.py b/ci/jobs/run_use_cases.py index 1ca795c1a3..23ffc7daad 100755 --- a/ci/jobs/run_use_cases.py +++ b/ci/jobs/run_use_cases.py @@ -69,13 +69,16 @@ def copy_diff_output(diff_files): and file path of output that was just generated. Either tuple value may be an empty string if the file was not found. """ - for truth_file, out_file, _ in diff_files: + for truth_file, out_file, _, diff_file in diff_files: if truth_file: copy_to_diff_dir(truth_file, 'truth') if out_file: copy_to_diff_dir(out_file, 'output') + if diff_file: + copy_to_diff_dir(diff_file, + 'diff') def copy_to_diff_dir(file_path, data_type): """! Generate output path based on input file path, @@ -143,7 +146,9 @@ def main(): if compare and isOK: print('******************************') print("Comparing output to truth data") - diff_files = compare_dir(TRUTH_DIR, OUTPUT_DIR, debug=True) + diff_files = compare_dir(TRUTH_DIR, OUTPUT_DIR, + debug=True, + save_diff=True) if diff_files: isOK = False diff --git a/ci/util/diff_util.py b/ci/util/diff_util.py index eefa0af3c3..157bc65142 100644 --- a/ci/util/diff_util.py +++ b/ci/util/diff_util.py @@ -54,10 +54,10 @@ def get_file_type(filepath): return 'unknown' -def compare_dir(dir_a, dir_b, debug=False): +def compare_dir(dir_a, dir_b, debug=False, save_diff=False): # if input are files and not directories, compare them if os.path.isfile(dir_a): - result = compare_files(dir_a, dir_b, debug=debug) + result = compare_files(dir_a, dir_b, debug=debug, save_diff=save_diff) if result is None or result is True: return [] @@ -89,7 +89,8 @@ def compare_dir(dir_a, dir_b, debug=False): filepath_b, debug=debug, dir_a=dir_a, - dir_b=dir_b) + dir_b=dir_b, + save_diff=save_diff) # no differences of skipped if result is None or result is True: @@ -108,13 +109,15 @@ def compare_dir(dir_a, dir_b, debug=False): filepath_a = filepath_b.replace(dir_b, dir_a) if not os.path.exists(filepath_a): print(f"ERROR: File does not exist: {filepath_a}") - diff_files.append(('', filepath_b, 'file not found (new output)')) + diff_files.append(('', filepath_b, 'file not found (new output)', '')) print("\nSummary:\n") if diff_files: print("\nERROR: Some differences were found") - for filepath_a, filepath_b, reason in diff_files: + for filepath_a, filepath_b, reason, diff_file in diff_files: print(f"{reason}\n A:{filepath_a}\n B:{filepath_b}") + if diff_file: + print(f"Difference file: {diff_file}") else: print("\nNo differences found in any files") @@ -122,7 +125,8 @@ def compare_dir(dir_a, dir_b, debug=False): "**************************************************\n\n") return diff_files -def compare_files(filepath_a, filepath_b, debug=False, dir_a=None, dir_b=None): +def compare_files(filepath_a, filepath_b, debug=False, dir_a=None, dir_b=None, + save_diff=False): # dir_a and dir_b are only needed if comparing file lists that need those # directories to substitute when comparing because files in the list will # have different paths @@ -133,7 +137,7 @@ def compare_files(filepath_a, filepath_b, debug=False, dir_a=None, dir_b=None): if not os.path.exists(filepath_b): if debug: print(f"ERROR: File does not exist: {filepath_b}") - return (filepath_a, '', 'file not found') + return (filepath_a, '', 'file not found', '') file_type = get_file_type(filepath_a) if file_type == 'skip': @@ -142,31 +146,41 @@ def compare_files(filepath_a, filepath_b, debug=False, dir_a=None, dir_b=None): if file_type.startswith('unsupported'): print(f"Unsupported file type encountered: {file_type.split('.')[1]}") - return (filepath_a, filepath_b, file_type) + return (filepath_a, filepath_b, file_type, '') if file_type == 'netcdf': print("Comparing NetCDF") if not nc_is_equal(filepath_a, filepath_b): - return (filepath_a, filepath_b, 'NetCDF diff') + return (filepath_a, filepath_b, 'NetCDF diff', '') print("No differences in NetCDF files") return True if file_type == 'pdf': print("Comparing PDF as images") - if not compare_pdf_as_images(filepath_a, filepath_b): - return (filepath_a, filepath_b, 'PDF diff') + diff_file = compare_pdf_as_images(filepath_a, filepath_b, + save_diff=save_diff) + if diff_file is True: + print("No differences in PDF files") + return True - print("No differences in PDF files") - return True + if diff_file is False: + diff_file = '' + + return (filepath_a, filepath_b, 'PDF diff', diff_file) if file_type == 'image': print("Comparing images") - if not compare_image_files(filepath_a, filepath_b): - return (filepath_a, filepath_b, 'Image diff') + diff_file = compare_image_files(filepath_a, filepath_b, + save_diff=save_diff) + if diff_file is True: + print("No differences in image files") + return True - print("No differences in image files") - return True + if diff_file is False: + diff_file = '' + + return (filepath_a, filepath_b, 'Image diff', diff_file) # if not any of the above types, use diff to compare print("Comparing text files") @@ -174,7 +188,7 @@ def compare_files(filepath_a, filepath_b, debug=False, dir_a=None, dir_b=None): # if files differ, open files and handle expected diffs if not compare_txt_files(filepath_a, filepath_b, dir_a, dir_b): print(f"ERROR: File differs: {filepath_b}") - return (filepath_a, filepath_b, 'Text diff') + return (filepath_a, filepath_b, 'Text diff', '') print("No differences in text files") return True @@ -183,28 +197,47 @@ def compare_files(filepath_a, filepath_b, debug=False, dir_a=None, dir_b=None): return True -def compare_pdf_as_images(filepath_a, filepath_b): +def compare_pdf_as_images(filepath_a, filepath_b, save_diff=False): try: from pdf2image import convert_from_path except ModuleNotFoundError: print("Cannot compare PDF files without pdf2image Python package") return False - all_equal = True images_a = convert_from_path(filepath_a) images_b = convert_from_path(filepath_b) for image_a, image_b in zip(images_a, images_b): - if not compare_images(image_a, image_b): - all_equal = False + image_diff = compare_images(image_a, image_b) + + # no differences if None, so continue to next image from PDF + if image_diff is None: + continue + + # if skipping save diff files, return False b/c there are differences + if not save_diff: + return False - return all_equal + # create difference image and return the path + return save_diff_file(image_diff, filepath_b) -def compare_image_files(filepath_a, filepath_b): + return True + +def compare_image_files(filepath_a, filepath_b, save_diff=False): image_a = Image.open(filepath_a) image_b = Image.open(filepath_b) - return compare_images(image_a, image_b) + image_diff = compare_images(image_a, image_b) + if image_diff is None: + return True + + if not save_diff: + return False + + return save_diff_file(image_diff, filepath_b) def compare_images(image_a, image_b): + """! Compare pillow image objects. Returns difference image object if there + are differences or None if not. + """ diff_count = 0 image_diff = ImageChops.difference(image_a, image_b) nx, ny = image_diff.size @@ -212,12 +245,19 @@ def compare_images(image_a, image_b): for y in range(0, int(ny)): pixel = image_diff.getpixel((x, y)) if pixel != 0 and pixel != (0, 0, 0, 0) and pixel != (0, 0, 0): + print(f"Difference pixel: {pixel}") diff_count += 1 if diff_count: print(f"ERROR: Found {diff_count} differences between images") - return False - return True - + return image_diff + return None + +def save_diff_file(image_diff, filepath_b): + rel_path, file_extension = os.path.splitext(filepath_b) + diff_file = f'{rel_path}_diff.png' + print(f"Saving diff file: {diff_file}") + image_diff.save(diff_file, "PNG") + return diff_file def compare_txt_files(filepath_a, filepath_b, dir_a=None, dir_b=None): with open(filepath_a, 'r') as file_handle: From 3ea362e5ae9d079c278f9c2d5d1dc7f0311296e5 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 17:45:16 -0600 Subject: [PATCH 31/43] ci-run-diff --- ci/util/diff_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/util/diff_util.py b/ci/util/diff_util.py index 157bc65142..c3df36921c 100644 --- a/ci/util/diff_util.py +++ b/ci/util/diff_util.py @@ -55,7 +55,7 @@ def get_file_type(filepath): return 'unknown' def compare_dir(dir_a, dir_b, debug=False, save_diff=False): - # if input are files and not directories, compare them + # if input are files and not directories, compare them if os.path.isfile(dir_a): result = compare_files(dir_a, dir_b, debug=debug, save_diff=save_diff) if result is None or result is True: From 5b021e136ce1d386ba34e90199c17b613a657744 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 18:30:14 -0600 Subject: [PATCH 32/43] install dependency of pdf2image --- ci/actions/run_tests/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/actions/run_tests/entrypoint.sh b/ci/actions/run_tests/entrypoint.sh index 0720217424..a80d1fd88b 100644 --- a/ci/actions/run_tests/entrypoint.sh +++ b/ci/actions/run_tests/entrypoint.sh @@ -48,7 +48,7 @@ fi # install Pillow library needed for diff testing # this will be replaced with better image diffing package used by METplotpy -pip_command="pip3 install Pillow; pip3 install pdf2image" +pip_command="pip3 install Pillow; pip3 install python-poppler; pip3 install pdf2image" # build command to run command="./ci/jobs/run_use_cases.py ${CATEGORIES} ${SUBSETLIST}" From b5066ead7836fb247fb99caf5edf0ad7baf8608a Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 18:31:35 -0600 Subject: [PATCH 33/43] ci-run-diff --- ci/actions/run_tests/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/actions/run_tests/entrypoint.sh b/ci/actions/run_tests/entrypoint.sh index a80d1fd88b..91d73b4945 100644 --- a/ci/actions/run_tests/entrypoint.sh +++ b/ci/actions/run_tests/entrypoint.sh @@ -1,6 +1,6 @@ #! /bin/bash -# The repo source code is cloned to $RUNNER_WORKSPACE/$REPO_NAME +# The repo source code is cloned to $RUNNER_WORKSPACE/$REPO_NAME # Setup the workspace path to that for easier access later REPO_NAME=$(basename $RUNNER_WORKSPACE) WS_PATH=$RUNNER_WORKSPACE/$REPO_NAME From 6e3424f83a49f0e3dfe2f6e2d5f9e284172edc2d Mon Sep 17 00:00:00 2001 From: George McCabe Date: Mon, 29 Mar 2021 19:01:08 -0600 Subject: [PATCH 34/43] install poppler-utils for pdf2image, ci-run-diff --- ci/actions/run_tests/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/actions/run_tests/entrypoint.sh b/ci/actions/run_tests/entrypoint.sh index 91d73b4945..2846a203a8 100644 --- a/ci/actions/run_tests/entrypoint.sh +++ b/ci/actions/run_tests/entrypoint.sh @@ -48,7 +48,7 @@ fi # install Pillow library needed for diff testing # this will be replaced with better image diffing package used by METplotpy -pip_command="pip3 install Pillow; pip3 install python-poppler; pip3 install pdf2image" +pip_command="pip3 install Pillow; sudo apt-get update -y; sudo apt-get install -y poppler-utils; pip3 install pdf2image" # build command to run command="./ci/jobs/run_use_cases.py ${CATEGORIES} ${SUBSETLIST}" From 9a6ffe5aa2460abb43b7e13efe0a470982d249c5 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 30 Mar 2021 06:47:37 -0600 Subject: [PATCH 35/43] fix install of poppler --- ci/actions/run_tests/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/actions/run_tests/entrypoint.sh b/ci/actions/run_tests/entrypoint.sh index 2846a203a8..7a43850a46 100644 --- a/ci/actions/run_tests/entrypoint.sh +++ b/ci/actions/run_tests/entrypoint.sh @@ -48,7 +48,7 @@ fi # install Pillow library needed for diff testing # this will be replaced with better image diffing package used by METplotpy -pip_command="pip3 install Pillow; sudo apt-get update -y; sudo apt-get install -y poppler-utils; pip3 install pdf2image" +pip_command="pip3 install Pillow; apt-get update -y; apt-get install -y poppler-utils; pip3 install pdf2image" # build command to run command="./ci/jobs/run_use_cases.py ${CATEGORIES} ${SUBSETLIST}" From 4daef230f818f9200e433d758133b059e0d6d409 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 30 Mar 2021 06:48:49 -0600 Subject: [PATCH 36/43] ci-run-diff --- ci/actions/run_tests/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/actions/run_tests/entrypoint.sh b/ci/actions/run_tests/entrypoint.sh index 7a43850a46..08a69e321b 100644 --- a/ci/actions/run_tests/entrypoint.sh +++ b/ci/actions/run_tests/entrypoint.sh @@ -1,6 +1,6 @@ #! /bin/bash -# The repo source code is cloned to $RUNNER_WORKSPACE/$REPO_NAME +# The repo source code is cloned to $RUNNER_WORKSPACE/$REPO_NAME # Setup the workspace path to that for easier access later REPO_NAME=$(basename $RUNNER_WORKSPACE) WS_PATH=$RUNNER_WORKSPACE/$REPO_NAME From bc96de2ff310e098c65838fb78abb31e8e9c3bc3 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 30 Mar 2021 07:18:22 -0600 Subject: [PATCH 37/43] use yum because docker image in centos --- ci/actions/run_tests/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/actions/run_tests/entrypoint.sh b/ci/actions/run_tests/entrypoint.sh index 08a69e321b..887bed54ac 100644 --- a/ci/actions/run_tests/entrypoint.sh +++ b/ci/actions/run_tests/entrypoint.sh @@ -48,7 +48,7 @@ fi # install Pillow library needed for diff testing # this will be replaced with better image diffing package used by METplotpy -pip_command="pip3 install Pillow; apt-get update -y; apt-get install -y poppler-utils; pip3 install pdf2image" +pip_command="pip3 install Pillow; yum install poppler-utils; pip3 install pdf2image" # build command to run command="./ci/jobs/run_use_cases.py ${CATEGORIES} ${SUBSETLIST}" From c9169ac641839a2a13129e0252c88b33738f67af Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 30 Mar 2021 07:26:58 -0600 Subject: [PATCH 38/43] try only mounting top level data dir instead of each subdir, ci-run-diff --- ci/actions/run_tests/entrypoint.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/actions/run_tests/entrypoint.sh b/ci/actions/run_tests/entrypoint.sh index 887bed54ac..6e43906e12 100644 --- a/ci/actions/run_tests/entrypoint.sh +++ b/ci/actions/run_tests/entrypoint.sh @@ -88,4 +88,5 @@ echo VOLUMES_FROM: $VOLUMES_FROM echo "Run Docker container: $DOCKERHUBTAG" echo docker run -e GITHUB_WORKSPACE -v $GHA_OUTPUT_DIR:$DOCKER_OUTPUT_DIR -v $GHA_DIFF_DIR:$DOCKER_DIFF_DIR -v $GHA_ERROR_LOG_DIR:$DOCKER_ERROR_LOG_DIR -v $WS_PATH:$GITHUB_WORKSPACE ${VOLUMES_FROM} --workdir $GITHUB_WORKSPACE $DOCKERHUBTAG bash -c "${pip_command};${command}" -docker run -e GITHUB_WORKSPACE -v $GHA_OUTPUT_DIR:$DOCKER_OUTPUT_DIR -v $GHA_DIFF_DIR:$DOCKER_DIFF_DIR -v $GHA_ERROR_LOG_DIR:$DOCKER_ERROR_LOG_DIR -v $WS_PATH:$GITHUB_WORKSPACE ${VOLUMES_FROM} --workdir $GITHUB_WORKSPACE $DOCKERHUBTAG bash -c "${pip_command};${command}" +#docker run -e GITHUB_WORKSPACE -v $GHA_OUTPUT_DIR:$DOCKER_OUTPUT_DIR -v $GHA_DIFF_DIR:$DOCKER_DIFF_DIR -v $GHA_ERROR_LOG_DIR:$DOCKER_ERROR_LOG_DIR -v $WS_PATH:$GITHUB_WORKSPACE ${VOLUMES_FROM} --workdir $GITHUB_WORKSPACE $DOCKERHUBTAG bash -c "${pip_command};${command}" +docker run -e GITHUB_WORKSPACE -v $RUNNER_WORKSPACE:$DOCKER_DATA_DIR -v $WS_PATH:$GITHUB_WORKSPACE ${VOLUMES_FROM} --workdir $GITHUB_WORKSPACE $DOCKERHUBTAG bash -c "${pip_command};${command}" From 48ea9869f518680a8e7c60adf38f37ea38e56702 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 30 Mar 2021 07:36:32 -0600 Subject: [PATCH 39/43] change back to mounting each output directory, ci-run-diff --- ci/actions/run_tests/entrypoint.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/actions/run_tests/entrypoint.sh b/ci/actions/run_tests/entrypoint.sh index 6e43906e12..887bed54ac 100644 --- a/ci/actions/run_tests/entrypoint.sh +++ b/ci/actions/run_tests/entrypoint.sh @@ -88,5 +88,4 @@ echo VOLUMES_FROM: $VOLUMES_FROM echo "Run Docker container: $DOCKERHUBTAG" echo docker run -e GITHUB_WORKSPACE -v $GHA_OUTPUT_DIR:$DOCKER_OUTPUT_DIR -v $GHA_DIFF_DIR:$DOCKER_DIFF_DIR -v $GHA_ERROR_LOG_DIR:$DOCKER_ERROR_LOG_DIR -v $WS_PATH:$GITHUB_WORKSPACE ${VOLUMES_FROM} --workdir $GITHUB_WORKSPACE $DOCKERHUBTAG bash -c "${pip_command};${command}" -#docker run -e GITHUB_WORKSPACE -v $GHA_OUTPUT_DIR:$DOCKER_OUTPUT_DIR -v $GHA_DIFF_DIR:$DOCKER_DIFF_DIR -v $GHA_ERROR_LOG_DIR:$DOCKER_ERROR_LOG_DIR -v $WS_PATH:$GITHUB_WORKSPACE ${VOLUMES_FROM} --workdir $GITHUB_WORKSPACE $DOCKERHUBTAG bash -c "${pip_command};${command}" -docker run -e GITHUB_WORKSPACE -v $RUNNER_WORKSPACE:$DOCKER_DATA_DIR -v $WS_PATH:$GITHUB_WORKSPACE ${VOLUMES_FROM} --workdir $GITHUB_WORKSPACE $DOCKERHUBTAG bash -c "${pip_command};${command}" +docker run -e GITHUB_WORKSPACE -v $GHA_OUTPUT_DIR:$DOCKER_OUTPUT_DIR -v $GHA_DIFF_DIR:$DOCKER_DIFF_DIR -v $GHA_ERROR_LOG_DIR:$DOCKER_ERROR_LOG_DIR -v $WS_PATH:$GITHUB_WORKSPACE ${VOLUMES_FROM} --workdir $GITHUB_WORKSPACE $DOCKERHUBTAG bash -c "${pip_command};${command}" From e685fdb45946e1dc65b9dc212cd966c1ea6ee8d5 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 30 Mar 2021 08:02:38 -0600 Subject: [PATCH 40/43] forced yum install, ci-run-diff --- ci/actions/run_tests/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/actions/run_tests/entrypoint.sh b/ci/actions/run_tests/entrypoint.sh index 887bed54ac..f34d3040e8 100644 --- a/ci/actions/run_tests/entrypoint.sh +++ b/ci/actions/run_tests/entrypoint.sh @@ -48,7 +48,7 @@ fi # install Pillow library needed for diff testing # this will be replaced with better image diffing package used by METplotpy -pip_command="pip3 install Pillow; yum install poppler-utils; pip3 install pdf2image" +pip_command="pip3 install Pillow; yum -y install poppler-utils; pip3 install pdf2image" # build command to run command="./ci/jobs/run_use_cases.py ${CATEGORIES} ${SUBSETLIST}" From f391b030985cbf191082e3e795d24164c9bf9724 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 30 Mar 2021 09:00:13 -0600 Subject: [PATCH 41/43] skip file only found in output (not truth) if it is a diff file that was generated by the run, ci-run-diff --- ci/util/diff_util.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/util/diff_util.py b/ci/util/diff_util.py index c3df36921c..ad3a7204ea 100644 --- a/ci/util/diff_util.py +++ b/ci/util/diff_util.py @@ -108,6 +108,10 @@ def compare_dir(dir_a, dir_b, debug=False, save_diff=False): filepath_b = os.path.join(root, filename) filepath_a = filepath_b.replace(dir_b, dir_a) if not os.path.exists(filepath_a): + # check if missing file is actually diff file that was generated + diff_list = [item[3] for item in diff_files] + if filepath_a in diff_list: + continue print(f"ERROR: File does not exist: {filepath_a}") diff_files.append(('', filepath_b, 'file not found (new output)', '')) From b7be3fe2fac6d3c8ea48398d4e5d7f34e980333b Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 30 Mar 2021 09:31:58 -0600 Subject: [PATCH 42/43] fixed check for diff file, ci-run-diff --- ci/util/diff_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/util/diff_util.py b/ci/util/diff_util.py index ad3a7204ea..05ea986f91 100644 --- a/ci/util/diff_util.py +++ b/ci/util/diff_util.py @@ -110,7 +110,7 @@ def compare_dir(dir_a, dir_b, debug=False, save_diff=False): if not os.path.exists(filepath_a): # check if missing file is actually diff file that was generated diff_list = [item[3] for item in diff_files] - if filepath_a in diff_list: + if filepath_b in diff_list: continue print(f"ERROR: File does not exist: {filepath_a}") diff_files.append(('', filepath_b, 'file not found (new output)', '')) From 7b4a72c46d607bfbe3c8683fab4437bf69b44e85 Mon Sep 17 00:00:00 2001 From: George McCabe Date: Tue, 30 Mar 2021 11:15:03 -0600 Subject: [PATCH 43/43] fixed logic so diff file does not have _diff_diff extension --- ci/jobs/run_use_cases.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ci/jobs/run_use_cases.py b/ci/jobs/run_use_cases.py index 23ffc7daad..80db5f8c59 100755 --- a/ci/jobs/run_use_cases.py +++ b/ci/jobs/run_use_cases.py @@ -100,8 +100,12 @@ def copy_to_diff_dir(file_path, data_type): diff_out = file_path.replace(data_dir, DIFF_DIR) # add data type identifier to filename before extension - output_path, extension = os.path.splitext(diff_out) - output_path = f'{output_path}_{data_type}{extension}' + # if data is not difference output + if data_type == 'diff': + output_path = diff_out + else: + output_path, extension = os.path.splitext(diff_out) + output_path = f'{output_path}_{data_type}{extension}' # create output directory if it doesn't exist output_dir = os.path.dirname(output_path)