Feature 1494 ens_stat (#1600)

Merging into the develop branch to get these changes included for the met-10.0.0-beta2 development release. These are good and worthwhile changes to make on their own. However, I do still need @j-opatz to confirm that they actually fix the original issue with python embedding in ensemble-stat!

* Per #1494, add DataPlane::is_all_bad_data() function to determine whether the field is all bad data.

* Per #1494, update the process_data_plane() function in 2 ways. First, change its return from void to bool. And simply return false, if the raw input data is all bad data. Second, print debug(4) log messages to list out the range of valid data and timing information. It'll be really nice to make this consistent across all the input file types.

* Per #1494, update the logic for all the calls to process_data_plane() to account for the fact that it now returns a bool instead of void.

* Per #1494, refine the Data plane debug(4) log messages.

* Per #1494, work on log message.

* Per #1494, updated vx_data2d_python library to call the common process_data_plane() function to apply data shifting, censoring, conversion, and check for all missing data.

* Per #1494, just fixing a small typo in an unrelated source code file that I ran across when testing for met-help.

met/src/basic/vx_util/data_plane.cc

@@ -279,6 +279,26 @@ void DataPlane::set_accum(int s) {
 
 ///////////////////////////////////////////////////////////////////////////////
 
+bool DataPlane::is_all_bad_data() const {
+   int j;
+   bool status = true;
+
+   //
+   // Check for no valid data
+   //
+
+   for(j=0; j<Nxy; ++j) {
+      if( !is_bad_data(Data[j]) ) {
+         status = false;
+         break;
+      }
+   }
+
+   return(status);
+}
+
+///////////////////////////////////////////////////////////////////////////////
+
 double DataPlane::get(int x, int y) const {
    int n;
 

met/src/basic/vx_util/data_plane.h

@@ -87,6 +87,8 @@ class DataPlane {
       int      nx() const;
       int      ny() const;
       bool     is_empty() const;
+      bool     is_all_bad_data() const;
+
       unixtime init() const;
       unixtime valid() const;
       int      lead() const;

met/src/libcode/vx_data2d/data_class.cc

@@ -309,11 +309,23 @@ return;
 ////////////////////////////////////////////////////////////////////////
 
 
-void Met2dDataFile::process_data_plane(VarInfo *vinfo, DataPlane &dp)
+bool Met2dDataFile::process_data_plane(VarInfo *vinfo, DataPlane &dp)
 
 {
 
-if ( ! vinfo )  return;
+if ( ! vinfo )  return ( false );
+
+   //
+   // Check for no valid input data
+   //
+
+if ( dp.is_all_bad_data() )  {
+
+   mlog << Debug(3) << "No valid data found in input data plane!\n";
+
+   return ( false );
+
+}
 
    //
    // Apply shift to the right logic
@@ -349,7 +361,26 @@ if ( vinfo->grid_attr().nxy() > 0 )  {
 
 }
 
-return;
+   //
+   // Print a data summary
+   //
+
+if ( mlog.verbosity_level() >= 4 ) {
+
+   double min_v, max_v;
+   dp.data_range(min_v, max_v);
+   mlog << Debug(4) << "\n"
+        << "Data plane information:\n"
+        << "      plane min: " << min_v << "\n"
+        << "      plane max: " << max_v << "\n"
+        << "     valid time: " << unix_to_yyyymmdd_hhmmss(dp.valid()) << "\n"
+        << "      lead time: " << sec_to_hhmmss(dp.lead()) << "\n"
+        << "      init time: " << unix_to_yyyymmdd_hhmmss(dp.init()) << "\n"
+        << "     accum time: " << sec_to_hhmmss(dp.accum()) << "\n\n";
+
+}
+
+return ( true );
 
 }
 

met/src/libcode/vx_data2d/data_class.h

@@ -155,7 +155,7 @@ class Met2dDataFile : public Met2dData {
 
          //  post-process data after reading it
 
-      void process_data_plane(VarInfo *, DataPlane &);
+      bool process_data_plane(VarInfo *, DataPlane &);
 
 };
 

met/src/libcode/vx_data2d_grib/data2d_grib.cc

@@ -536,6 +536,8 @@ int MetGrib1DataFile::data_plane_array(VarInfo &vinfo,
             rotate_winds(vinfo_grib_winds, cur_plane);
          }
 
+         if(status) status = process_data_plane(&vinfo, cur_plane);
+
          if(!status) {
             cur_plane.clear();
             lower = upper = bad_data_int;
@@ -544,9 +546,6 @@ int MetGrib1DataFile::data_plane_array(VarInfo &vinfo,
                  << " from GRIB file \"" << filename() << "\".\n\n";
             continue;
          }
-         else {
-            process_data_plane(&vinfo, cur_plane);
-         }
 
          // Add current record to the data plane array
          plane_array.add(cur_plane, (double) lower, (double) upper);
@@ -712,7 +711,7 @@ bool MetGrib1DataFile::data_plane_scalar(VarInfoGrib &vinfo_grib,
          // Read current record
          status = get_data_plane(r, plane);
 
-         if(status) process_data_plane(&vinfo_grib, plane);
+         if(status) status = process_data_plane(&vinfo_grib, plane);
 
          break;
       }

met/src/libcode/vx_data2d_grib2/data2d_grib2.cc

@@ -191,9 +191,7 @@ bool MetGrib2DataFile::data_plane(VarInfo &vinfo, DataPlane &plane) {
 
       plane = plane_array[0];
 
-      process_data_plane(vinfo_g2, plane);
-
-      return true;
+      return(process_data_plane(vinfo_g2, plane));
 
    }  //  END: if( 1 > listMatch.size() )
 
@@ -235,7 +233,7 @@ bool MetGrib2DataFile::data_plane(VarInfo &vinfo, DataPlane &plane) {
    //  check the data plane for wind rotation
    plane = check_uv_rotation(vinfo_g2, listMatch[0], plane);
 
-   if(read_success) process_data_plane(vinfo_g2, plane);
+   if(read_success) read_success = process_data_plane(vinfo_g2, plane);
 
    return read_success;
 }
@@ -350,9 +348,9 @@ int MetGrib2DataFile::data_plane_array( VarInfo &vinfo,
          lvl_upper = ( (double)(*it)->LvlVal2 ) / 100.0;
       }
 
-      process_data_plane(vinfo_g2, plane);
-
-      plane_array.add(plane, lvl_lower, lvl_upper);
+      if(process_data_plane(vinfo_g2, plane)) {
+         plane_array.add(plane, lvl_lower, lvl_upper);
+      }
    }
 
    return num_read;

met/src/libcode/vx_data2d_nc_met/data2d_nc_met.cc

@@ -183,7 +183,7 @@ bool MetNcMetDataFile::data_plane(VarInfo &vinfo, DataPlane &plane)
          status = false;
       }
 
-      process_data_plane(&vinfo, plane);
+      status = process_data_plane(&vinfo, plane);
 
       // Set the VarInfo object's name, long_name, level, and units strings
       if(info->name_att.length()      > 0) vinfo.set_name(info->name_att);

met/src/libcode/vx_data2d_nc_pinterp/data2d_nc_pinterp.cc

@@ -162,7 +162,7 @@ bool MetNcPinterpDataFile::data_plane(VarInfo &vinfo, DataPlane &plane) {
          status = false;
       }
 
-      process_data_plane(&vinfo, plane);
+      status = process_data_plane(&vinfo, plane);
 
       // Set the VarInfo object's name, long_name, and units strings
       if(info->name_att.length()      > 0) vinfo.set_name(info->name_att);
@@ -255,7 +255,7 @@ int MetNcPinterpDataFile::data_plane_array(VarInfo &vinfo,
             status = false;
          }
 
-         process_data_plane(&vinfo, cur_plane);
+         status = process_data_plane(&vinfo, cur_plane);
 
          // Add current plane to the data plane array
          plane_array.add(cur_plane, pressure, pressure);

met/src/libcode/vx_data2d_nccf/data2d_nccf.cc

@@ -212,7 +212,7 @@ bool MetNcCFDataFile::data_plane(VarInfo &vinfo, DataPlane &plane)
           do_stop = false;
           dimension[time_dim_slot] = 0;
           mlog << Warning << "\n" << method_name << "returns the first available time for \""
-               << vinfo.req_name() << "\" variable).\n\n";
+               << vinfo.req_name() << "\" variable.\n\n";
         }
         else
           mlog << Warning << "\n" << method_name << "the requested offset "
@@ -270,7 +270,7 @@ bool MetNcCFDataFile::data_plane(VarInfo &vinfo, DataPlane &plane)
       status = false;
     }
 
-    process_data_plane(&vinfo, plane);
+    status = process_data_plane(&vinfo, plane);
 
     // Set the VarInfo object's name, long_name, level, and units strings
 
@@ -287,22 +287,8 @@ bool MetNcCFDataFile::data_plane(VarInfo &vinfo, DataPlane &plane)
 
     if (info->units_att.length() > 0)
       vinfo.set_units(info->units_att.c_str());
-
-    //  print a report
-    double plane_min, plane_max;
-    plane.data_range(plane_min, plane_max);
-    mlog << Debug(4) << "\n"
-         << "Data plane information:\n"
-         << "      plane min: " << plane_min << "\n"
-         << "      plane max: " << plane_max << "\n"
-         << "     valid time: " << unix_to_yyyymmdd_hhmmss(plane.valid()) << "\n"
-         << "      lead time: " << sec_to_hhmmss(plane.lead()) << "\n"
-         << "      init time: " << unix_to_yyyymmdd_hhmmss(plane.init()) << "\n"
-         << "     accum time: " << sec_to_hhmmss(plane.accum()) << "\n";
   }
 
-
-
   return status;
 }
 

met/src/libcode/vx_data2d_python/data2d_python.cc

@@ -360,6 +360,8 @@ if ( PythonCommand.empty() || PythonCommand != vinfo.req_name() ) {
 
 }
 
+if ( status )  status = process_data_plane(&vinfo, Plane);
+
 if ( !status )  return ( false );
 
    //
@@ -419,6 +421,8 @@ if ( PythonCommand.empty() || PythonCommand != vinfo.req_name() ) {
 
 }
 
+if ( status )  status = process_data_plane(&vinfo, Plane);
+
 if ( !status )  return ( 0 );
 
    //