Update develop-ref after #2056, #2057, and #2060 (#2059)

Co-authored-by: Julie Prestopnik <[email protected]> Co-authored-by: johnhg <[email protected]> Co-authored-by: Seth Linden <[email protected]> Co-authored-by: John Halley Gotway <[email protected]> Co-authored-by: j-opatz <[email protected]> Co-authored-by: Howard Soh <[email protected]> Co-authored-by: John Halley Gotway <[email protected]> Co-authored-by: jprestop <[email protected]> Co-authored-by: Howard Soh <[email protected]> Co-authored-by: Seth Linden <[email protected]> Co-authored-by: hsoh-u <[email protected]> Co-authored-by: George McCabe <[email protected]> Co-authored-by: John Halley Gotway <[email protected]> Co-authored-by: MET Tools Test Account <[email protected]> Co-authored-by: mo-mglover <[email protected]> Co-authored-by: davidalbo <[email protected]> Co-authored-by: lisagoodrich <[email protected]>
dtcenter · Feb 20, 2022 · e9499f1 · e9499f1
1 parent ee1fb0d
commit e9499f1
Show file tree

Hide file tree

Showing 32 changed files with 497 additions and 298 deletions.
diff --git a/.github/dummy_for_action b/.github/dummy_for_action
@@ -0,0 +1,2 @@
+#update me to add action comment
+
diff --git a/.github/jobs/build_docker_image.sh b/.github/jobs/build_docker_image.sh
@@ -12,3 +12,7 @@ time_command docker build -t ${DOCKERHUB_TAG} \
     --build-arg SOURCE_BRANCH \
     --build-arg MET_BASE_IMAGE \
     -f $DOCKERFILE_PATH ${GITHUB_WORKSPACE}
+if [ $? != 0 ]; then
+  cat ${GITHUB_WORKSPACE}/docker_build.log
+  exit 1
+fi
diff --git a/met/data/config/ConfigConstants b/met/data/config/ConfigConstants
@@ -88,6 +88,7 @@ AW_MEAN        = 20;
 GAUSSIAN       = 21;
 MAXGAUSS       = 22;
 GEOG_MATCH     = 23;
+HIRA           = 24;
 
 // Interpolation types
 NONE           = 1;

diff --git a/met/data/config/PB2NCConfig_default b/met/data/config/PB2NCConfig_default
@@ -107,7 +107,7 @@ obs_bufr_map = [];
 // abbreviations in the output.  This default map is appended to obs_bufr_map.
 // This should not typically be overridden.
 //
-obs_prefbufr_map = [
+obs_prepbufr_map = [
    { key = "POB";     val = "PRES";  },
    { key = "QOB";     val = "SPFH";  },
    { key = "TOB";     val = "TMP";   },
@@ -121,7 +121,8 @@ obs_prefbufr_map = [
    { key = "D_MIXR";  val = "MIXR";  },
    { key = "D_PRMSL"; val = "PRMSL"; },
    { key = "D_PBL";   val = "PBL";   },
-   { key = "D_CAPE";  val = "CAPE";  }
+   { key = "D_CAPE";  val = "CAPE";  },
+   { key = "D_MLCAPE";  val = "MLCAPE";  }
 ];
 
 ////////////////////////////////////////////////////////////////////////////////

diff --git a/met/docs/Users_Guide/config_options.rst b/met/docs/Users_Guide/config_options.rst
@@ -610,7 +610,7 @@ using the following entries:
     * MAXGAUSS    to compute the maximum value in the neighborhood
       and apply a Gaussian smoother to the result
 
-    The BEST and GEOG_MATCH interpolation options are not valid for regridding.
+    The BEST, GEOG_MATCH, and HIRA options are not valid for regridding.
 
 * The "width" entry specifies a regridding width, when applicable.
   - width = 4;    To regrid using a 4x4 box or circle with diameter 4.
@@ -1689,7 +1689,10 @@ This dictionary may include the following entries:
     * MAXGAUSS    for the maximum value followed by a Gaussian smoother
 
     * GEOG_MATCH  for the nearest grid point where the land/sea mask
-      and geography criteria are satisfied.
+      and geography criteria are satisfied
+
+    * HIRA        for all neighborhood points to define a spatial
+      ensemble (only in Ensemble-Stat)
 
     The BUDGET, FORCE, GAUSSIAN, and MAXGAUSS methods are not valid for
     interpolating to point locations. For grid-to-grid comparisons, the
@@ -3467,7 +3470,7 @@ of the forecast the observation is used to verify.
 		
   obs_bufr_map = [];
 
-obs_prefbufr_map
+obs_prepbufr_map
 """"""""""""""""
 
 Default mapping for PREPBUFR. Replace input BUFR variable names with GRIB
@@ -3478,7 +3481,7 @@ abbreviations to the output.
 
 .. code-block:: none
 		
-  obs_prefbufr_map = [
+  obs_prepbufr_map = [
      { key = "POB";     val = "PRES";  },
      { key = "QOB";     val = "SPFH";  },
      { key = "TOB";     val = "TMP";   },

diff --git a/met/docs/Users_Guide/ensemble-stat.rst b/met/docs/Users_Guide/ensemble-stat.rst
@@ -17,6 +17,8 @@ Ensemble forecasts derived from a set of deterministic ensemble members
 
 Ensemble forecasts are often created as a set of deterministic forecasts. The ensemble members are rarely used separately. Instead, they can be combined in various ways to produce a forecast. MET can combine the ensemble members into some type of summary forecast according to user specifications. Ensemble means are the most common, and can be paired with the ensemble variance or spread. Maximum, minimum and other summary values are also available, with details in the practical information section.
 
+Typically an ensemble is constructed by selecting a single forecast value from each member for each observation. When the High Resolution Assessment (HiRA) interpolation method is chosen, all of the nearby neighborhood points surrounding each observation from each member are used. Therefore, processing an N-member ensemble using a HiRA neighborhood of size M produces ensemble output with size N*M. This approach fully leverages information from all nearby grid points to evaluate the ensemble quality.
+
 The ensemble relative frequency is the simplest method for turning a set of deterministic forecasts into something resembling a probability forecast. MET will create the ensemble relative frequency as the proportion of ensemble members forecasting some event. For example, if 5 out of 10 ensemble members predict measurable precipitation at a grid location, then the ensemble relative frequency of precipitation will be :math:`5/10=0.5`. If the ensemble relative frequency is calibrated (unlikely) then this could be thought of as a probability of precipitation.
 
 The neighborhood ensemble probability (NEP) and neighborhood maximum ensemble probability (NMEP) methods are described in :ref:`Schwartz and Sobash (2017) <Schwartz-2017>`. They are an extension of the ensemble relative frequencies described above. The NEP value is computed by averaging the relative frequency of the event within the neighborhood over all ensemble members. The NMEP value is computed as the fraction of ensemble members for which the event is occurring somewhere within the surrounding neighborhood. The NMEP output is typically smoothed using a Gaussian kernel filter. The neighborhood sizes and smoothing options can be customized in the configuration file.
@@ -161,6 +163,8 @@ ____________________
 
 The configuration options listed above are common to many MET tools and are described in :numref:`config_options`.
 
+Note that the **HIRA** interpolation method is only supported in Ensemble-Stat.
+
 _____________________
 
 .. code-block:: none

diff --git a/met/docs/Users_Guide/reformat_point.rst b/met/docs/Users_Guide/reformat_point.rst
@@ -227,7 +227,7 @@ _____________________
   obs_bufr_var = [ 'QOB', 'TOB', 'ZOB', 'UOB', 'VOB' ];
 
 
-Each PrepBUFR message will likely contain multiple observation variables. The **obs_bufr_var** variable is used to specify which observation variables should be retained or derived. The variable name comes from BUFR file which includes BUFR table. The following BUFR names may be retained: QOB, TOB, ZOB, UOB, and VOB for specific humidity, temperature, height, and the u and v components of winds. The following BUFR names may be derived: D_DPT, D_WIND, D_RH, D_MIXR, D_PRMSL, D_PBL, and D_CAPE for dew point, wind speed, relative humidity, mixing ratio, pressure reduced to MSL, planetary boundary layer height, and convective available potential energy. This configuration replaces **obs_grib_code**. If the list is empty, all BUFR variables are retained.
+Each PrepBUFR message will likely contain multiple observation variables. The **obs_bufr_var** variable is used to specify which observation variables should be retained or derived. The variable name comes from BUFR file which includes BUFR table. The following BUFR names may be retained: QOB, TOB, ZOB, UOB, and VOB for specific humidity, temperature, height, and the u and v components of winds. The following BUFR names may be derived: D_DPT, D_WIND, D_RH, D_MIXR, D_PRMSL, D_PBL, D_CAPE, and D_MLCAPE for dew point, wind speed, relative humidity, mixing ratio, pressure reduced to MSL, planetary boundary layer height, convective available potential energy, and mixed layer convective available potential energy. This configuration replaces **obs_grib_code**. If the list is empty, all BUFR variables are retained.
 
 _____________________
 
@@ -248,6 +248,7 @@ _____________________
 		{ key = 'D_PRMSL';  val = 'PRMSL'; },
 		{ key = 'D_PBL';    val = 'PBL';   },
 		{ key = 'D_CAPE';   val = 'CAPE';  }
+		{ key = 'D_MLCAPE'; val = 'MLCAPE';  }
 		];
 
 

diff --git a/met/scripts/config/EnsembleStatConfig b/met/scripts/config/EnsembleStatConfig
@@ -249,6 +249,10 @@ interp = {
       {
          method = NEAREST;
          width  = 1;
+      },
+      {
+         method = HIRA;
+         width  = 2;
       }
    ];
 }

diff --git a/met/scripts/config/PB2NCConfig_G212 b/met/scripts/config/PB2NCConfig_G212
@@ -84,7 +84,7 @@ obs_bufr_var = [ "QOB", "TOB", "ZOB", "UOB", "VOB",
 // Mapping of BUFR variable name to GRIB name. The default map is defined at
 // obs_prepbufr_map. This replaces/expends the default map.
 //
-//obs_bufr_var = [];
+obs_bufr_map = [];
 
 ////////////////////////////////////////////////////////////////////////////////
 

diff --git a/met/scripts/config/STATAnalysisConfig b/met/scripts/config/STATAnalysisConfig
@@ -84,7 +84,7 @@ jobs = [
    "-job aggregate      -line_type GRAD                                          -vx_mask DTC165   -vx_mask DTC166 -by FCST_VAR      -dump_row ${TEST_OUT_DIR}/stat_analysis/job_aggregate_GRAD.stat",
    "-job aggregate      -line_type ISC                        -fcst_thresh >0.0  -vx_mask TILE_TOT                 -fcst_var APCP_12 -dump_row ${TEST_OUT_DIR}/stat_analysis/job_aggregate_ISC.stat",
    "-job aggregate      -line_type RHIST                      -obtype MC_PCP     -vx_mask HUC4_1605 -vx_mask HUC4_1803 -vx_mask HUC4_1804 -vx_mask HUC4_1805 -vx_mask HUC4_1806 -dump_row ${TEST_OUT_DIR}/stat_analysis/job_aggregate_RHIST.stat",
-   "-job aggregate_stat -line_type ORANK -out_line_type RHIST -obtype ADPSFC     -vx_mask HUC4_1605 -vx_mask HUC4_1803 -vx_mask HUC4_1804 -vx_mask HUC4_1805 -vx_mask HUC4_1806 -dump_row ${TEST_OUT_DIR}/stat_analysis/job_aggregate_stat_ORANK_RHIST.stat"
+   "-job aggregate_stat -line_type ORANK -out_line_type RHIST -obtype ADPSFC     -vx_mask HUC4_1605,HUC4_1803,HUC4_1804,HUC4_1805,HUC4_1806 -by INTERP_MTHD,INTERP_PNTS -dump_row ${TEST_OUT_DIR}/stat_analysis/job_aggregate_stat_ORANK_RHIST.stat"
 ];
 
 ////////////////////////////////////////////////////////////////////////////////

diff --git a/met/src/basic/vx_config/config_constants.h b/met/src/basic/vx_config/config_constants.h
@@ -571,7 +571,7 @@ static const char conf_key_message_type_group_map[] = "message_type_group_map";
 static const char conf_key_obs_bufr_map[]      = "obs_bufr_map";
 static const char conf_key_obs_bufr_var[]      = "obs_bufr_var";
 static const char conf_key_obs_name_map[]      = "obs_name_map";
-static const char conf_key_obs_prefbufr_map[]  = "obs_prefbufr_map";
+static const char conf_key_obs_prepbufr_map[]  = "obs_prepbufr_map";
 static const char conf_key_key[]               = "key";
 static const char conf_key_val[]               = "val";
 static const char conf_key_boot_interval[]     = "boot.interval";

diff --git a/met/src/basic/vx_config/config_util.cc b/met/src/basic/vx_config/config_util.cc
@@ -24,6 +24,7 @@ using namespace std;
 ///////////////////////////////////////////////////////////////////////////////
 
 static const double default_vld_thresh = 1.0;
+static const char conf_key_prepbufr_map_bad[] = "obs_prefbufr_map";    // for backward compatibility
 
 ///////////////////////////////////////////////////////////////////////////////
 
@@ -140,9 +141,10 @@ RegridInfo::RegridInfo() {
 void RegridInfo::validate() {
 
    // Check for unsupported regridding options
-   if(method == InterpMthd_Best ||
+   if(method == InterpMthd_Best       ||
       method == InterpMthd_Geog_Match ||
-      method == InterpMthd_Gaussian) {
+      method == InterpMthd_Gaussian   ||
+      method == InterpMthd_HiRA) {
       mlog << Error << "\nRegridInfo::validate() -> "
            << "\"" << interpmthd_to_string(method)
            << "\" not valid for regridding, only interpolating.\n\n";
@@ -1088,14 +1090,6 @@ map<ConcatString,StringArray> parse_conf_metadata_map(Dictionary *dict) {
 
 ///////////////////////////////////////////////////////////////////////////////
 
-map<ConcatString,ConcatString> parse_conf_obs_bufr_map(Dictionary *dict) {
-   map<ConcatString,ConcatString> m = parse_conf_key_value_map(dict, conf_key_obs_prefbufr_map);
-   parse_add_conf_key_value_map(dict, conf_key_obs_bufr_map, &m);
-   return m;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-
 map<ConcatString,ConcatString> parse_conf_obs_name_map(Dictionary *dict) {
    const char *method_name = "parse_conf_obs_name_map() -> ";
    return parse_conf_key_value_map(dict, conf_key_obs_name_map);
@@ -2279,6 +2273,7 @@ InterpMthd int_to_interpmthd(int i) {
    else if(i == conf_const.lookup_int(interpmthd_gaussian_str))    m = InterpMthd_Gaussian;
    else if(i == conf_const.lookup_int(interpmthd_maxgauss_str))    m = InterpMthd_MaxGauss;
    else if(i == conf_const.lookup_int(interpmthd_geog_match_str))  m = InterpMthd_Geog_Match;
+   else if(i == conf_const.lookup_int(interpmthd_hira_str))        m = InterpMthd_HiRA;
    else {
       mlog << Error << "\nconf_int_to_interpmthd() -> "
            << "Unexpected value of " << i

diff --git a/met/src/basic/vx_config/config_util.h b/met/src/basic/vx_config/config_util.h
@@ -25,6 +25,10 @@
 
 ////////////////////////////////////////////////////////////////////////
 
+static const char conf_key_old_prepbufr_map[] = "obs_prefbufr_map";    // for backward compatibility
+
+////////////////////////////////////////////////////////////////////////
+
 extern ConcatString    parse_conf_version(Dictionary *dict);
 extern ConcatString    parse_conf_string(Dictionary *dict, const char *, bool check_empty = true);
 extern GrdFileType     parse_conf_file_type(Dictionary *dict);
@@ -48,13 +52,13 @@ extern ClimoCDFInfo    parse_conf_climo_cdf(Dictionary *dict);
 extern TimeSummaryInfo parse_conf_time_summary(Dictionary *dict);
 extern map<ConcatString,ConcatString> parse_conf_key_value_map(
                             Dictionary *dict, const char *conf_key_map_name, const char *caller=0);
+extern void            parse_add_conf_key_value_map(
+                            Dictionary *dict, const char *conf_key_map_name, map<ConcatString,ConcatString> *m);
 extern map<ConcatString,ConcatString>
                        parse_conf_message_type_map(Dictionary *dict);
 extern map<ConcatString,StringArray>
                        parse_conf_message_type_group_map(Dictionary *dict);
 extern map<ConcatString,StringArray> parse_conf_metadata_map(Dictionary *dict);
-extern map<ConcatString,ConcatString>
-                       parse_conf_obs_bufr_map(Dictionary *dict);
 extern map<ConcatString,ConcatString>
                        parse_conf_obs_name_map(Dictionary *dict);
 extern BootInfo        parse_conf_boot(Dictionary *dict);

diff --git a/met/src/basic/vx_util/interp_mthd.cc b/met/src/basic/vx_util/interp_mthd.cc
@@ -44,6 +44,7 @@ ConcatString interpmthd_to_string(const InterpMthd m) {
       case(InterpMthd_Gaussian):    out = interpmthd_gaussian_str;    break;
       case(InterpMthd_MaxGauss):    out = interpmthd_maxgauss_str;    break;
       case(InterpMthd_Geog_Match):  out = interpmthd_geog_match_str;  break;
+      case(InterpMthd_HiRA):        out = interpmthd_hira_str;        break;
 
       case(InterpMthd_None):
       default:                      out = interpmthd_none_str;        break;
@@ -77,6 +78,7 @@ InterpMthd string_to_interpmthd(const char *mthd_str) {
    else if(strcmp(mthd_str, interpmthd_gaussian_str  )  == 0) m = InterpMthd_Gaussian;
    else if(strcmp(mthd_str, interpmthd_maxgauss_str  )  == 0) m = InterpMthd_MaxGauss;
    else if(strcmp(mthd_str, interpmthd_geog_match_str)  == 0) m = InterpMthd_Geog_Match;
+   else if(strcmp(mthd_str, interpmthd_hira_str)        == 0) m = InterpMthd_HiRA;
    else                                                       m = InterpMthd_None;
 
    return(m);

diff --git a/met/src/basic/vx_util/interp_mthd.h b/met/src/basic/vx_util/interp_mthd.h
@@ -42,7 +42,8 @@ enum InterpMthd {
    InterpMthd_Lower_Left,
    InterpMthd_Gaussian,
    InterpMthd_MaxGauss,
-   InterpMthd_Geog_Match
+   InterpMthd_Geog_Match,
+   InterpMthd_HiRA
 };
 
 //
@@ -69,6 +70,7 @@ static const char interpmthd_lower_left_str[]  = "LOWER_LEFT";
 static const char interpmthd_gaussian_str[]    = "GAUSSIAN";
 static const char interpmthd_maxgauss_str[]    = "MAXGAUSS";
 static const char interpmthd_geog_match_str[]  = "GEOG_MATCH";
+static const char interpmthd_hira_str[]        = "HIRA";
 
 ///////////////////////////////////////////////////////////////////////////////
 

diff --git a/met/src/basic/vx_util/interp_util.cc b/met/src/basic/vx_util/interp_util.cc
@@ -1020,6 +1020,7 @@ double compute_sfc_interp(const DataPlane &dp,
          mlog << Error << "\ncompute_sfc_interp() -> "
               << "unsupported interpolation method encountered: "
               << interpmthd_to_string(mthd) << "(" << mthd << ")\n\n";
+         exit(1);
    }
 
    delete gt;

diff --git a/met/src/libcode/vx_data2d/var_info.h b/met/src/libcode/vx_data2d/var_info.h
@@ -258,9 +258,9 @@ inline int          VarInfo::accum_attr()     const { return(SetAttrAccum);    }
 //
 
 struct InputInfo {
-    VarInfo * var_info;         // Variable information to read
-    int file_index;             // Index in file_list of file to read
-    StringArray * file_list;    // Array of files (unallocated)
+   VarInfo * var_info;      // Variable information to read
+   int file_index;          // Index in file_list of file to read
+   StringArray * file_list; // Array of files (unallocated)
 };
 
 ////////////////////////////////////////////////////////////////////////
@@ -270,33 +270,37 @@ struct InputInfo {
 //
 class EnsVarInfo {
 
-private:
-    vector<InputInfo> inputs;     // Vector of InputInfo
-    VarInfo * ctrl_info;          // Field info for control member
-public:
-    EnsVarInfo();
-    ~EnsVarInfo();
-    EnsVarInfo(const EnsVarInfo &);
+   private:
+      vector<InputInfo> inputs; // Vector of InputInfo
+      VarInfo * ctrl_info;      // Field info for control member
 
-    void clear();
-    void assign(const EnsVarInfo &);
+   public:
+      EnsVarInfo();
+      ~EnsVarInfo();
+      EnsVarInfo(const EnsVarInfo &);
+
+      void clear();
+      void assign(const EnsVarInfo &);
 
-    void add_input(InputInfo);
-    int inputs_n();
+      void add_input(InputInfo);
+      int inputs_n();
 
-    void set_ctrl(VarInfo *);
-    VarInfo * get_ctrl(int);
+      void set_ctrl(VarInfo *);
+      VarInfo * get_ctrl(int);
 
-    // Get VarInfo from first InputInfo if requested VarInfo is NULL
-    VarInfo * get_var_info(int index=0);
-    ConcatString get_file(int index=0);
-    int get_file_index(int index=0);
+      // Get VarInfo from first InputInfo if requested VarInfo is NULL
+      VarInfo * get_var_info(int index=0);
+      ConcatString get_file(int index=0);
+      int get_file_index(int index=0);
+
+      ConcatString nc_var_str;      // Ensemble variable name strings
+      ThreshArray cat_ta;           // Ensemble categorical thresholds
+      ConcatString raw_magic_str;   // Magic string w/o var substitution
 
-    ConcatString nc_var_str;      // Ensemble variable name strings
-    ThreshArray cat_ta;           // Ensemble categorical thresholds
-    ConcatString raw_magic_str;   // Magic string w/o var substitution
 };
 
+////////////////////////////////////////////////////////////////////////
+
 ConcatString raw_magic_str(Dictionary i_edict, GrdFileType file_type);
 
 ///////////////////////////////////////////////////////////////////////////////

diff --git a/met/src/libcode/vx_data2d_nc_met/met_file.cc b/met/src/libcode/vx_data2d_nc_met/met_file.cc
@@ -330,8 +330,6 @@ void MetNcFile::dump(ostream & out, int depth) const
 {
 
 int j, k;
-int month, day, year, hour, minute, second;
-char junk[256];
 Indent prefix(depth);
 Indent p2(depth + 1);
 Indent p3(depth + 2);
@@ -362,12 +360,6 @@ out << prefix << "Ydim = " << (Ydim ? GET_NC_NAME_P(Ydim) : "(nul)") << "\n";
 
 out << prefix << "\n";
 
-snprintf(junk, sizeof(junk), "%s %d, %d   %2d:%02d:%02d", short_month_name[month], day, year, hour, minute, second);
-
-out << junk << "\n";
-
-out << prefix << "\n";
-
 out << prefix << "Nvars = " << Nvars << "\n";
 
 for (j=0; j<Nvars; ++j)  {

diff --git a/met/src/libcode/vx_nc_obs/nc_obs_util.h b/met/src/libcode/vx_nc_obs/nc_obs_util.h
@@ -98,7 +98,7 @@ struct NetcdfObsVars {
    NcDim obs_arr_dim ; // Observation array width (V1.0, not used from V1.2)
    NcDim obs_dim     ; // Observation array length (V1.0)
    NcDim hdr_dim     ; // Header array length (V1.0)
-   NcDim pb_hdr_dim  ; // PrefBufr Header array length (V1.2)
+   NcDim pb_hdr_dim  ; // PrepfBufr Header array length (V1.2)
 
    NcVar hdr_typ_tbl_var ; // Message type (string) (V1.1)
    NcVar hdr_sid_tbl_var ; // Station ID (string) (V1.1)