Update main_v9.1-ref for the nightly build. (#1566)

* Add debug level 4 message to list out the number of GRIB2 records inventoried. This helps debugging issues with MET potentially not reading all input GRIB2 records on WCOSS.

* Bugfix 1554 main_v9.1 ncdump (#1555)

* Bugfix 1562 main_v9.1 grid_diag (#1563)

* Per #1562, check for bad data values before adding data to the PDF's for grid_diag.

* Per #1562, removing the poly = CONUS.poly mask from GridDiagConfig_TMP. That settting masked a problem in the handling of missing data. Exercising the mask.poly option is tested in another unit test. This will change the output and break the nightly build, but that's good since we'll do more thorough testing.

* Per #1508, change the verbosity in unit_tc_gen.xml from -v 2 to -v 5 to print out some additional log messages that may help in debugging the intermittent file list failure.

met/src/libcode/vx_data2d_grib2/data2d_grib2.cc

@@ -930,6 +930,10 @@ void MetGrib2DataFile::read_grib2_record_list() {
 
    }  //  END:  while( read_grib2_record() )
 
+   mlog << Debug(4) << "MetGrib2DataFile::read_grib2_record_list() - "
+        << "processed " << rec_num - 1 << " records from GRIB2 file: "
+        << Filename << "\n";
+
    return;
 }
 

met/src/libcode/vx_series_data/series_pdf.cc

@@ -66,8 +66,9 @@ void update_pdf(
 
     for(int i = 0; i < dp.nx(); i++) {
         for(int j = 0; j < dp.ny(); j++) {
-            if(!mp.s_is_on(i, j)) continue;
             double value = dp.get(i, j);
+            if(!mp.s_is_on(i, j) ||
+               is_bad_data(value)) continue;
             int k = floor((value - min) / delta);
             if(k < 0) k = 0;
             if(k >= pdf.size()) k = pdf.size() - 1;
@@ -92,12 +93,14 @@ void update_joint_pdf(
 
     for(int i = 0; i < dp_A.nx(); i++) {
         for(int j = 0; j < dp_A.ny(); j++) {
-            if(!mp.s_is_on(i, j)) continue;
             double value_A = dp_A.get(i, j);
+            double value_B = dp_B.get(i, j);
+            if(!mp.s_is_on(i, j)    ||
+               is_bad_data(value_A) ||
+               is_bad_data(value_B)) continue;
             int k_A = floor((value_A - min_A) / delta_A);
             if(k_A < 0) k_A = 0;
             if(k_A >= n_A) k_A = n_A - 1;
-            double value_B = dp_B.get(i, j);
             int k_B = floor((value_B - min_B) / delta_B);
             if(k_B < 0) k_B = 0;
             if(k_B >= n_B) k_B = n_B - 1;

scripts/docker/Dockerfile

@@ -131,7 +131,7 @@ RUN echo "Downloading HDF4.2r3 from ${HDF4_URL}" \
  && cd /met/external_libs/HDF4.2r3 \
  && LOG_FILE=/met/logs/HDF4.2r3_configure.log \
  && echo "Configuring HDF4.2r3 and writing log file ${LOG_FILE}" \
- && ./configure --prefix=/usr/local --disable-netcdf > ${LOG_FILE} \
+ && ./configure --prefix=/usr/local/hdf --disable-netcdf > ${LOG_FILE} \
  && cat mfhdf/hdiff/Makefile | sed 's/LIBS = -ljpeg -lz/LIBS = -ljpeg -lz -lm/g' > Makefile_NEW \
  && mv -f Makefile_NEW mfhdf/hdiff/Makefile \
  && LOG_FILE=/met/logs/HDF4.2r3_make_install.log \
@@ -142,11 +142,12 @@ RUN echo "Downloading HDF4.2r3 from ${HDF4_URL}" \
  && cd /met/external_libs/hdfeos \
  && LOG_FILE=/met/logs/hdfeos_configure.log \
  && echo "Configuring hdfeos and writing log file ${LOG_FILE}" \
- && ./configure --prefix=/usr/local --with-hdf4=/usr/local CC=/usr/local/bin/h4cc > ${LOG_FILE} \
+ && ./configure --prefix=/usr/local/hdfeos --with-hdf4=/usr/local/hdf CC=/usr/local/hdf/bin/h4cc > ${LOG_FILE} \
  && LOG_FILE=/met/logs/hdfeos_make_install.log \
  && echo "Compiling hdfeos and writing log file ${LOG_FILE}" \
  && make install > ${LOG_FILE} \
- && cp include/*.h /usr/local/include/. \
+ && mkdir /usr/local/hdfeos/include \
+ && cp include/*.h /usr/local/hdfeos/include/. \
  && cd /met/external_libs \
  && rm -rf HDF4.2r3 hdfeos
 
@@ -163,6 +164,7 @@ RUN echo "Checking out MET ${MET_GIT_NAME} from ${MET_GIT_URL}" \
  && ./bootstrap \
  && echo "Configuring MET ${MET_GIT_NAME} and writing log file ${LOG_FILE}" \
  && ./configure --enable-grib2 --enable-mode_graphics --enable-modis --enable-lidar2nc --enable-python \
+    MET_HDF=/usr/local/hdf MET_HDFEOS=/usr/local/hdfeos \
     MET_FREETYPEINC=/usr/include/freetype2 MET_FREETYPELIB=/usr/lib \
     MET_CAIROINC=/usr/include/cairo MET_CAIROLIB=/usr/lib \
     MET_PYTHON_CC='-I/usr/include/python3.6m' MET_PYTHON_LD='-lpython3.6m' > ${LOG_FILE} \

test/config/GridDiagConfig_TMP

@@ -62,7 +62,7 @@ data = {
 
 mask = {
    grid = "";
-   poly = "MET_BASE/poly/CONUS.poly";
+   poly = "";
 }
 
 ////////////////////////////////////////////////////////////////////////////////

test/xml/unit_tc_gen.xml

@@ -22,7 +22,7 @@
       -config &CONFIG_DIR;/TCGenConfig_2016 \
       -out &OUTPUT_DIR;/tc_gen/tc_gen_2016 \
       -log &OUTPUT_DIR;/tc_gen/tc_gen_2016.log \
-      -v 2
+      -v 5 
     </param>
     <output>
       <stat>&OUTPUT_DIR;/tc_gen/tc_gen_2016.stat</stat>