Feature #2924 parse_config (#2963)

* Per #2924, remove GenEnsProd config file comment about parsing desc separately from each obs.field entry because the obs dictionary does not exist in the GenEnsProd config file.

* Per #2924, update list of needed config entry names

* Per #2924, remove const from the parent() member function so that we can perform lookups for the parent.

* Per #2924, update the signature for and logic of the utility functions that retrieve the climatology data. Rather than requiring all the climo_mean and climo_stdev dictionary entries to be defined at the same config file context level, parse each one individually. This enables the METplus wrappers to only partially override this dictionary and still rely on the default values provided in MET's default configuration files.

* Per #2924, update all calls to the climatology utility functions based on the new function signature. Also update the tools to check the number of climo fields separately for the forecast and observation climos.

* Per #2924, update the parsing logic for the climatology regrid dictionary. Use config.fcst.climo_mean.regrid first, config.fcst.regrid second, and config.climo_mean.regrid third. Notably, DO NOT use config.regrid. This is definitely the problem with having regrid specified at mutliple config file context levels. It makes the logic for which to use when very messy.

* Per #2924, forgot to add an else to print an error

* Per #2924, remove extraneous semicolon

* Per #2924, move 'fcst.regrid' into 'fcst.climo_mean.regrid'. Defining the climatology regridding logic inside fcst is problematic because it applies to the forecast data as well and you end up with the verification grid being undefined. So the climo regridding logic must be defined in 'climo_mean.regrid' either within the 'fcst' and 'obs' dictionaries or at the top-level config context.

* Per #2924, based on PR feedback from @georgemccabe, add the Upper_Left, Upper_Right, Lower_Right, and Lower_Left interpolation methods to the list of valid options for regridding, as already indicated in the MET User's Guide.

* Per #2924, update the logic of parse_conf_regrid() to (hopefully) make it work the way @georgemccabe expects it to. It now uses pointers to both the primary and default dictionaries and parses each entry individually.

* Per #2924, need to check for non-null pointer before using it

* Per #2924, revise the climo_name dictionary lookup logic when parsing the regrid dictionary.

* Per #2924, update logic for handling RegridInfo

* Per #2924, remove the default regridding information from the 'Searching' log message to avoid confusion.

---------

Co-authored-by: MET Tools Test Account <[email protected]>

data/config/GenEnsProdConfig_default

@@ -13,7 +13,6 @@ model = "FCST";
 
 //
 // Output description to be written
-// May be set separately in each "obs.field" entry
 //
 desc = "NA";
 

internal/test_unit/config/GenEnsProdConfig

@@ -13,7 +13,6 @@ model = "FCST";
 
 //
 // Output description to be written
-// May be set separately in each "obs.field" entry
 //
 desc = "NA";
 

internal/test_unit/config/GenEnsProdConfig_climo_anom_ens_member_id

@@ -13,7 +13,6 @@ model = "CFSv2";
 
 //
 // Output description to be written
-// May be set separately in each "obs.field" entry
 //
 desc = "NA";
 

internal/test_unit/config/GenEnsProdConfig_normalize

@@ -13,7 +13,6 @@ model = "FCST";
 
 //
 // Output description to be written
-// May be set separately in each "obs.field" entry
 //
 desc = "NA";
 

internal/test_unit/config/GenEnsProdConfig_single_file_grib

@@ -13,7 +13,6 @@ model = "GEFS";
 
 //
 // Output description to be written
-// May be set separately in each "obs.field" entry
 //
 desc = "NA";
 

internal/test_unit/config/GenEnsProdConfig_single_file_nc

@@ -13,7 +13,6 @@ model = "CFSv2";
 
 //
 // Output description to be written
-// May be set separately in each "obs.field" entry
 //
 desc = "NA";
 

internal/test_unit/config/GridStatConfig_climo_FCST_NCEP_1.0DEG_OBS_WMO_1.5DEG

@@ -75,16 +75,24 @@ fcst = {
    climo_mean = {
       field = field_list;
       file_name = [ "${FCST_CLIMO_DIR}/cmean_1d.19590410" ];
+
+      regrid = {
+         method     = BILIN;
+         width      = 2;
+         vld_thresh = 0.5;
+         shape      = SQUARE;
+      }
+
+      time_interp_method = DW_MEAN;
+      day_interval       = 1;
+      hour_interval      = 6;
    };
 
+   climo_stdev = climo_mean;
    climo_stdev = {
-      field = field_list;
       file_name = [ "${FCST_CLIMO_DIR}/cstdv_1d.19590410" ];
    };
 
-   time_interp_method = DW_MEAN;
-   day_interval       = 1;
-   hour_interval      = 6;
 }
 
 obs = {
@@ -99,18 +107,24 @@ obs = {
                     "${OBS_CLIMO_DIR}/u850hPa_mean.grib",
                     "${OBS_CLIMO_DIR}/v500hPa_mean.grib",
                     "${OBS_CLIMO_DIR}/v850hPa_mean.grib" ];
+      regrid = {
+         method     = BILIN;
+         width      = 2;
+         vld_thresh = 0.5;
+         shape      = SQUARE;
+      }
+
+      time_interp_method = DW_MEAN;
+      day_interval       = 1;
+      hour_interval      = 12;
    };
 
+   climo_stdev = climo_mean;
    climo_stdev = {
-      field = field_list;
       file_name = [ "${OBS_CLIMO_DIR}/t850hPa_stdev.grib",
                     "${OBS_CLIMO_DIR}/u850hPa_stdev.grib",
                     "${OBS_CLIMO_DIR}/v850hPa_stdev.grib" ];
    };
-
-   time_interp_method = DW_MEAN;
-   day_interval       = 1;
-   hour_interval      = 12;
 }
 
 ////////////////////////////////////////////////////////////////////////////////

scripts/config/GenEnsProdConfig

@@ -13,7 +13,6 @@ model = "FCST";
 
 //
 // Output description to be written
-// May be set separately in each "obs.field" entry
 //
 desc = "NA";
 

src/basic/vx_config/config_constants.h

@@ -297,7 +297,7 @@ struct InterpInfo {
    void        clear();
    void        validate(); // Ensure that width and method are accordant
    bool        operator==(const InterpInfo &) const;
-   InterpInfo &operator=(const InterpInfo &a) noexcept; // SoanrQube findings
+   InterpInfo &operator=(const InterpInfo &a) noexcept; // SonarQube findings
 };
 
 ////////////////////////////////////////////////////////////////////////
@@ -329,6 +329,7 @@ struct RegridInfo {
    void validate();        // ensure that width and method are accordant
    void validate_point();  // ensure that width and method are accordant
    RegridInfo &operator=(const RegridInfo &a) noexcept; // SoanrQube findings
+   ConcatString get_str() const;
 };
 
 ////////////////////////////////////////////////////////////////////////
@@ -725,12 +726,10 @@ static const char conf_key_is_prob[]              = "is_prob";
 //
 // Climatology data parameter key names
 //
-static const char conf_key_climo_mean_field[]       = "climo_mean.field";
-static const char conf_key_fcst_climo_mean_field[]  = "fcst.climo_mean.field";
-static const char conf_key_obs_climo_mean_field[]   = "obs.climo_mean.field";
-static const char conf_key_climo_stdev_field[]      = "climo_stdev.field";
-static const char conf_key_fcst_climo_stdev_field[] = "fcst.climo_stdev.field";
-static const char conf_key_obs_climo_stdev_field[]  = "obs.climo_stdev.field";
+static const char conf_key_climo_mean[]        = "climo_mean";
+static const char conf_key_climo_mean_field[]  = "climo_mean.field";
+static const char conf_key_climo_stdev[]       = "climo_stdev";
+static const char conf_key_climo_stdev_field[] = "climo_stdev.field";
 
 //
 // Climatology distribution parameter key names

src/basic/vx_config/config_util.cc

@@ -14,6 +14,7 @@
 
 #include "config_util.h"
 #include "enum_as_int.hpp"
+#include "configobjecttype_to_string.h"
 
 #include "vx_math.h"
 #include "vx_util.h"
@@ -265,6 +266,13 @@ RegridInfo &RegridInfo::operator=(const RegridInfo &a) noexcept {
    return *this;
 }
 
+///////////////////////////////////////////////////////////////////////////////
+
+ConcatString RegridInfo::get_str() const {
+   ConcatString cs(interpmthd_to_string(method));
+   cs << "(" << width << ")";
+   return cs;
+}
 
 ///////////////////////////////////////////////////////////////////////////////
 
@@ -1331,13 +1339,10 @@ BootInfo parse_conf_boot(Dictionary *dict) {
    return info;
 }
 
-
 ///////////////////////////////////////////////////////////////////////////////
 
-RegridInfo parse_conf_regrid(Dictionary *dict, bool error_out) {
-   Dictionary *regrid_dict = (Dictionary *) nullptr;
+RegridInfo parse_conf_regrid(Dictionary *dict, RegridInfo *default_info, bool error_out) {
    RegridInfo info;
-   int v;
 
    if(!dict) {
       mlog << Error << "\nparse_conf_regrid() -> "
@@ -1346,10 +1351,10 @@ RegridInfo parse_conf_regrid(Dictionary *dict, bool error_out) {
    }
 
    // Conf: regrid
-   regrid_dict = dict->lookup_dictionary(conf_key_regrid, false);
+   Dictionary *regrid_dict = dict->lookup_dictionary(conf_key_regrid, false);
 
    // Check that the regrid dictionary is present
-   if(!regrid_dict) {
+   if(!regrid_dict && !default_info) {
       if(error_out) {
          mlog << Error << "\nparse_conf_regrid() -> "
               << "can't find the \"regrid\" dictionary!\n\n";
@@ -1360,61 +1365,164 @@ RegridInfo parse_conf_regrid(Dictionary *dict, bool error_out) {
       }
    }
 
-   // Parse to_grid as an integer
-   v = regrid_dict->lookup_int(conf_key_to_grid, false, false);
+   // Conf: to_grid (optional) as an integer or string
+   const DictionaryEntry * entry = nullptr;
+
+   if(regrid_dict) entry = regrid_dict->lookup(conf_key_to_grid, false);
+
+   // to_grid found
+   if(entry) {
 
-   // If integer lookup successful, convert to FieldType.
-   if(regrid_dict->last_lookup_status()) {
-      info.field  = int_to_fieldtype(v);
-      info.enable = (info.field == FieldType::Fcst ||
-                     info.field == FieldType::Obs);
+      // Convert integer to FieldType
+      if(entry->type() == IntegerType) {
+         info.field  = int_to_fieldtype(entry->i_value());
+         info.enable = (info.field == FieldType::Fcst ||
+                        info.field == FieldType::Obs);
+      }
+      // Store grid name string
+      else if(entry->type() == StringType) {
+         info.name   = entry->string_value();
+         info.enable = true;
+      }
+      else {
+         mlog << Error << "\nparse_conf_regrid() -> "
+              << "Unexpected type ("
+              << configobjecttype_to_string(entry->type())
+              << ") for \"" << conf_key_to_grid
+              << "\" configuration entry.\n\n";
+         exit(1);
+      }
+   }
+   // Use default RegridInfo
+   else if(default_info){
+      info.name   = default_info->name;
+      info.enable = default_info->enable;
    }
-   // If integer lookup unsuccessful, parse vx_grid as a string.
-   // Do not error out since to_grid isn't specified for climo.regrid.
+   // Use global default
    else {
-      info.name   = regrid_dict->lookup_string(conf_key_to_grid, false);
+      info.name   = "";
       info.enable = true;
    }
 
-   // Conf: vld_thresh
-   double thr      = regrid_dict->lookup_double(conf_key_vld_thresh, false);
-   info.vld_thresh = (is_bad_data(thr) ? default_vld_thresh : thr);
+   // Conf: vld_thresh (required)
+   if(regrid_dict && regrid_dict->lookup(conf_key_vld_thresh, false)) {
+      info.vld_thresh = regrid_dict->lookup_double(conf_key_vld_thresh);
+   }
+   // Use default RegridInfo
+   else if(default_info) {
+      info.vld_thresh = default_info->vld_thresh;
+   }
+   // Use global default
+   else {
+      info.vld_thresh = default_vld_thresh;
+   }
+
+   // Conf: method (required)
+   if(regrid_dict && regrid_dict->lookup(conf_key_method, false)) {
+      info.method = int_to_interpmthd(regrid_dict->lookup_int(conf_key_method));
+   }
+   // Use default RegridInfo
+   else if(default_info) {
+      info.method = default_info->method;
+   }
 
-   // Parse the method and width
-   info.method = int_to_interpmthd(regrid_dict->lookup_int(conf_key_method));
-   info.width  = regrid_dict->lookup_int(conf_key_width);
+   // Conf: width (required)
+   if(regrid_dict && regrid_dict->lookup(conf_key_width, false)) {
+      info.width = regrid_dict->lookup_int(conf_key_width);
+   }
+   // Use default RegridInfo
+   else if(default_info) {
+      info.width = default_info->width;
+   }
 
-   // Conf: shape
-   v = regrid_dict->lookup_int(conf_key_shape, false);
-   if (regrid_dict->last_lookup_status()) {
-      info.shape = int_to_gridtemplate(v);
+   // Conf: shape (optional)
+   if(regrid_dict && regrid_dict->lookup(conf_key_shape, false)) {
+      info.shape = int_to_gridtemplate(regrid_dict->lookup_int(conf_key_shape));
+   }
+   // Use default RegridInfo
+   else if(default_info) {
+      info.shape = default_info->shape;
    }
+   // Use global default
    else {
-      // If not specified, use the default square shape
       info.shape = GridTemplateFactory::GridTemplates::Square;
    }
 
-   // Conf: gaussian dx and radius
-   double conf_value = regrid_dict->lookup_double(conf_key_gaussian_dx, false);
-   info.gaussian.dx = (is_bad_data(conf_value) ? default_gaussian_dx : conf_value);
-   conf_value = regrid_dict->lookup_double(conf_key_gaussian_radius, false);
-   info.gaussian.radius = (is_bad_data(conf_value) ? default_gaussian_radius : conf_value);
-   conf_value = regrid_dict->lookup_double(conf_key_trunc_factor, false);
-   info.gaussian.trunc_factor = (is_bad_data(conf_value) ? default_trunc_factor : conf_value);
-   if (info.method == InterpMthd::Gaussian || info.method == InterpMthd::MaxGauss) info.gaussian.compute();
+   // Conf: gaussian_dx (optional)
+   if(regrid_dict && regrid_dict->lookup(conf_key_gaussian_dx, false)) {
+      info.gaussian.dx = regrid_dict->lookup_double(conf_key_gaussian_dx);
+   }
+   // Use default RegridInfo
+   else if(default_info) {
+      info.gaussian.dx = default_info->gaussian.dx;
+   }
+   // Use global default
+   else {
+      info.gaussian.dx = default_gaussian_dx;
+   }
+
+   // Conf: gaussian_radius (optional)
+   if(regrid_dict && regrid_dict->lookup(conf_key_gaussian_radius, false)) {
+      info.gaussian.radius = regrid_dict->lookup_double(conf_key_gaussian_radius);
+   }
+   // Use default RegridInfo
+   else if(default_info) {
+      info.gaussian.radius = default_info->gaussian.radius;
+   }
+   // Use global default
+   else {
+      info.gaussian.radius = default_gaussian_radius;
+   }
+
+   // Conf: gaussian_trunc_factor (optional)
+   if(regrid_dict && regrid_dict->lookup(conf_key_trunc_factor, false)) {
+      info.gaussian.trunc_factor = regrid_dict->lookup_double(conf_key_trunc_factor);
+   }
+   // Use default RegridInfo
+   else if(default_info) {
+      info.gaussian.trunc_factor = default_info->gaussian.trunc_factor;
+   }
+   // Use global default
+   else {
+      info.gaussian.trunc_factor = default_trunc_factor;
+   }
+
+   // Compute Guassian parameters
+   if(info.method == InterpMthd::Gaussian ||
+      info.method == InterpMthd::MaxGauss) {
+      info.gaussian.compute();
+   }
 
    // MET#2437 Do not search the higher levels of config file context for convert,
    //          censor_thresh, and censor_val. They must be specified within the
    //          regrid dictionary itself.
 
-   // Conf: convert
-   info.convert_fx.set(regrid_dict->lookup(conf_key_convert, false));
+   // Conf: convert (optional)
+   if(regrid_dict && regrid_dict->lookup(conf_key_convert, false)) {
+      info.convert_fx.set(regrid_dict->lookup(conf_key_convert));
+   }
+   // Use default RegridInfo
+   else if(default_info) {
+      info.convert_fx = default_info->convert_fx;
+   }
 
-   // Conf: censor_thresh
-   info.censor_thresh = regrid_dict->lookup_thresh_array(conf_key_censor_thresh, false, true, false);
+   // Conf: censor_thresh (optional)
+   if(regrid_dict && regrid_dict->lookup(conf_key_censor_thresh, false)) {
+      info.censor_thresh = regrid_dict->lookup_thresh_array(conf_key_censor_thresh);
+   }
+   // Use default RegridInfo
+   else if(default_info) {
+      info.censor_thresh = default_info->censor_thresh;
+   }
 
-   // Conf: censor_val
-   info.censor_val = regrid_dict->lookup_num_array(conf_key_censor_val, false, true, false);
+   // Conf: censor_val (optional)
+   if(regrid_dict && regrid_dict->lookup(conf_key_censor_val, false)) {
+      info.censor_val = regrid_dict->lookup_num_array(conf_key_censor_val);
+   }
+   // Use default RegridInfo
+   else if(default_info) {
+      info.censor_val = default_info->censor_val;
+   }
 
    // Validate the settings
    info.validate();