Feature 1950 sort station list (#2121)

* Update the top-level README file to list ioda2nc, but also to trigger GHA to run for this new main_v10.1-ref branch. * Added bool Sorted variable, set to true or false in relevant functions. Updated sort() function. SL * In parse_sid_mask(), added the line: mask_sid.sort(). SL * Per issue #1950, put in some cout statements for testing. SL * Per issue #1950, working in has() function, checking for Sort and put in initial cout's for testing. SL * Per issue #1950: Cleaned up some cout (print) statements. Left one in there that is commented out for future testing. SL * Per issue #1950: modified has() function. Added lower_bound search for sorted arrays. Left some print (cout) statements in there for further testing. SL * Per issue #1950: In has(), I commented out the print (cout) statements for time testing. SL * added logic to manually trigger a workflow via the GitHub… (#2107) * Per issue #1950, put back in some print (cout) statements for testing. SL * Per issue #1950: in has() function, experimenting using the upper_bound() search instead of lower_bound() for sorted vectors. In progress. SL * Per issue #1950: added a new has function that uses binary_search. Re-working the other has() functions related to this. SL * Per issue #1950: re-worked original has(), basically reverted back to what it was...but it's now only used for unsorted vectors or for a case insensitive search. Cleaned up. SL * Per issue #1950: in section that checks Obs Station Id in SID masking list, changed cout to a mlog (debug) statement. SL * Per issue #1950: created new mask station id list (CONUS METARs) and modified the PointStatConfig_MASK_SID config file to use this list along with the existing lists. SL * Per issue #1950: reverted back to orginal sid mask site-lists. SL * Per issue #1950: checking in this site-list that was used for testing point_stat. SL * Per issue #1950, modified and added better debugging to figure out current issue with ascii2nc. SL * Per #1950, small tweak in ascii2nc to make it slightly more efficient. After adding the new obs varable name to the end of the list, we can compute var_index direclty without needing to call StringArray::has() a second time. * Per #1950: removed mlog and cout print statements from has() functions. Cleaned up. SL * Update met/src/basic/vx_log/string_array.cc Co-authored-by: johnhg <[email protected]> * Update met/src/basic/vx_log/string_array.cc Co-authored-by: johnhg <[email protected]> * Update met/src/basic/vx_log/string_array.cc Co-authored-by: johnhg <[email protected]> * Update met/src/basic/vx_log/string_array.cc Co-authored-by: johnhg <[email protected]> * Update met/src/basic/vx_log/string_array.cc Co-authored-by: johnhg <[email protected]> * Update met/src/basic/vx_log/string_array.h Co-authored-by: johnhg <[email protected]> * Delete SID_CONUS.txt * Delete SID_CONUS_times11.txt Co-authored-by: John Halley Gotway <[email protected]> Co-authored-by: Seth Linden <[email protected]> Co-authored-by: George McCabe <[email protected]>
dtcenter · Apr 6, 2022 · cf7ddec · cf7ddec
1 parent a320761
commit cf7ddec
Show file tree

Hide file tree

Showing 5 changed files with 128 additions and 80 deletions.
diff --git a/met/src/basic/vx_config/config_util.cc b/met/src/basic/vx_config/config_util.cc
@@ -690,6 +690,9 @@ void parse_sid_mask(const ConcatString &mask_sid_str,
 
    }
 
+   // Sort the mask_sid's
+   mask_sid.sort();
+
    return;
 }
 

diff --git a/met/src/basic/vx_log/string_array.cc b/met/src/basic/vx_log/string_array.cc
@@ -135,6 +135,8 @@ MaxLength = 0;
 
 clear();
 
+Sorted = false;
+
 return;
 
 }
@@ -149,6 +151,8 @@ void StringArray::clear()
 
 s.clear();
 
+Sorted = false;
+
 return;
 
 }
@@ -166,6 +170,7 @@ clear();
 s = a.s;
 
 IgnoreCase = a.IgnoreCase;
+Sorted = a.Sorted;
 
 return;
 
@@ -183,6 +188,7 @@ Indent prefix(depth);
 Indent prefix2(depth + 1);
 
 out << prefix << "IgnoreCase = " << IgnoreCase << "\n";
+out << prefix << "Sorted     = " << (Sorted ? "true" : "false") << "\n";
 
 int j;
 
@@ -250,6 +256,8 @@ void StringArray::add(const std::string text)
 
 s.push_back(text);
 
+Sorted = false;
+
 return;
 
 }
@@ -265,7 +273,9 @@ void StringArray::add(const StringArray & a)
 if ( a.n() == 0 )  return;
 
 s.insert(s.end(), a.s.begin(), a.s.end());
-
+
+Sorted = false;
+
 return;
 
 }
@@ -298,6 +308,8 @@ void StringArray::add_css(const std::string text)
 
   }
 
+  Sorted = false;
+
   return;
 
 }
@@ -314,6 +326,9 @@ s.clear();
 
 s.push_back(text);
 
+// Setting to a single value, by nature it is Sorted
+Sorted = true;
+
 return;
 
 }
@@ -336,6 +351,8 @@ void StringArray::set(int i, const std::string text)
 
 s[i] = text;
 
+Sorted = false;
+
 return;
 
 }
@@ -358,6 +375,8 @@ void StringArray::insert(int i, const char * text)
 
   s.insert(s.begin()+i, text);
 
+  Sorted = false;
+
   return;
 
 }
@@ -366,90 +385,104 @@ void StringArray::insert(int i, const char * text)
 ////////////////////////////////////////////////////////////////////////
 
 
-bool StringArray::has(const std::string text, bool forward) const
+bool StringArray::has(const std::string text) const
 
 {
+   bool found = false;
+   bool forward = true;
+
+   if (Sorted && !IgnoreCase) {
+      found = binary_search(s.begin(), s.end(), text);
+}
+else {
+      return ( has(text, forward) );
+}
 
-  int index;
+   return found;
+}
 
-  return ( has(text, index, forward) );
 
-}
+////////////////////////////////////////////////////////////////////////
+
+
+bool StringArray::has(const std::string text, bool forward) const
 
+{
+   int index;
+   return ( has(text, index, forward) );
+}
 
 ////////////////////////////////////////////////////////////////////////
 
 
 bool StringArray::has(const std::string text, int & index, bool forward) const
-
 {
-  bool found = false;
-  index = -1;
-
-  if (!s.empty()) {
-    int count;
-    std::string lower_text = text;
-    std::vector<std::string>::const_iterator it;
-    if ( IgnoreCase ) transform(lower_text.begin(), lower_text.end(), lower_text.begin(), ::tolower);
-
-    if (forward) {
-      count = 0;
-      for(it = s.begin(); it != s.end(); it++, count++) {
-        if ( IgnoreCase ) {
-          std::string lower_s = *it;
-          transform(lower_s.begin(), lower_s.end(), lower_s.begin(), ::tolower);
-          if ( lower_s == lower_text) {
-            //      if ( strcasecmp((*it).c_str(), text.c_str()) ) {
-            found = true;
-            break;
-          }
-        }
-        else {
-          if ( *it == text ) {
-            found = true;
-            break;
-          }
-        }
-      }
-    }
-    else {
-      count = s.size() - 1;
-      it = s.end();
-      for(it--; it != s.begin(); it--, count--) {
-        if ( IgnoreCase ) {
-          std::string lower_s = *it;
-          transform(lower_s.begin(), lower_s.end(), lower_s.begin(), ::tolower);
-          if ( lower_s == lower_text) {
-            found = true;
-            break;
-          }
-        }
-        else {
-          if ( *it == text ) {
-            found = true;
-            break;
-          }
-        }
+   // This function is now used for either an un-sorted array (Sorted is false)
+   // Or for a case-insensitive search (IgnoreCase is true)
+   //
+   bool found = false;
+   index = -1;
+
+   if (!s.empty()) {
+      int count;
+      std::string lower_text = text;
+      std::vector<std::string>::const_iterator it;
+      if ( IgnoreCase ) transform(lower_text.begin(), lower_text.end(), lower_text.begin(), ::tolower);
+
+      if (forward) {
+         count = 0;
+         for(it = s.begin(); it != s.end(); it++, count++) {
+            if ( IgnoreCase ) {
+               std::string lower_s = *it;
+               transform(lower_s.begin(), lower_s.end(), lower_s.begin(), ::tolower);
+               if ( lower_s == lower_text) {
+                  found = true;
+                  break;
+               }
+            }
+            else {
+               if ( *it == text ) {
+                  found = true;
+                  break;
+               }
+            }
+         }
       }
-      if (!found && it == s.begin()) {
-        if ( IgnoreCase ) {
-          std::string lower_s = *it;
-          transform(lower_s.begin(), lower_s.end(), lower_s.begin(), ::tolower);
-          found = ( lower_s == lower_text );
-        }
-        else found = ( *it == text );
+      else {
+         count = s.size() - 1;
+         it = s.end();
+         for(it--; it != s.begin(); it--, count--) {
+            if ( IgnoreCase ) {
+               std::string lower_s = *it;
+               transform(lower_s.begin(), lower_s.end(), lower_s.begin(), ::tolower);
+               if ( lower_s == lower_text) {
+                  found = true;
+                  break;
+               }
+            }
+            else {
+               if ( *it == text ) {
+                  found = true;
+                  break;
+               }
+            }
+         }
+         if (!found && it == s.begin()) {
+            if ( IgnoreCase ) {
+               std::string lower_s = *it;
+               transform(lower_s.begin(), lower_s.end(), lower_s.begin(), ::tolower);
+               found = ( lower_s == lower_text );
+            }
+            else found = ( *it == text );
+         }
       }
-    }
-    if (found) index = count;
-  }
-  //mlog << Debug(9) << " StringArray::has() size=" << s.size()
-  //     << " for " << text << ", found: " << (found ? "true" : "false")
-  //     << ", forward: " << (forward ? "yes" : "no") << "\n";
-
-  return found;
-
+      if (found) index = count;
+   }
+
+   return found;
 }
 
+
 ////////////////////////////////////////////////////////////////////////
 
 
@@ -500,6 +533,8 @@ void StringArray::parse_delim(const std::string text, const char *delim)
   if (start < str.length())
       s.push_back(str.substr(start).c_str());
 
+  Sorted = false;
+
   return;
 
 }
@@ -616,13 +651,19 @@ return ( s[k].length() );
 void StringArray::sort()
 
 {
-
-if ( n() <= 1 )  return;
-
- std::sort(s.begin(), s.end());
-
-return;
-
+   if ( n() <= 1 ) {
+      Sorted = true;
+      return;
+   }
+
+   if ( !Sorted ) {
+      std::sort(s.begin(), s.end());
+   }
+
+   Sorted = true;
+
+   return;
+
 }
 
 

diff --git a/met/src/basic/vx_log/string_array.h b/met/src/basic/vx_log/string_array.h
@@ -30,7 +30,6 @@ class StringArray {
 
    public:
 
-
       void init_from_scratch();
 
       void assign(const StringArray &);
@@ -40,7 +39,8 @@ class StringArray {
       int  MaxLength;
 
       bool IgnoreCase;
-
+
+      bool Sorted;
 
    public:
 
@@ -77,8 +77,10 @@ class StringArray {
 
       int length(int) const;
 
-      bool has(const std::string, bool forward=true) const;
+      bool has(const std::string) const;
 
+      bool has(const std::string, bool forward) const;
+
       bool has(const std::string, int & index, bool forward=true) const;
 
          //

diff --git a/met/src/libcode/vx_statistics/pair_data_point.cc b/met/src/libcode/vx_statistics/pair_data_point.cc
@@ -1142,6 +1142,8 @@ void VxPairDataPoint::add_point_obs(float *hdr_arr, const char *hdr_typ_str,
          // masking SID list
          else if(pd[i][j][0].mask_sid_ptr != (StringArray *) 0) {
             if(!pd[i][j][0].mask_sid_ptr->has(hdr_sid_str)) {
+               mlog << Debug(9) << "Checking for the obs station id in the masking SID list: rejected hdr_sid_str = "
+                    << hdr_sid_str << "\n";
                inc_count(rej_mask, i, j);
                continue;
             }

diff --git a/met/src/tools/other/ascii2nc/met_handler.cc b/met/src/tools/other/ascii2nc/met_handler.cc
@@ -159,7 +159,7 @@ bool MetHandler::_readObservations(LineDataFile &ascii_file)
         use_var_id = true;
         if (!obs_names.has(data_line[6], var_index)) {
            obs_names.add(data_line[6]);
-           obs_names.has(data_line[6], var_index);
+           var_index = obs_names.n() - 1;
         }
         grib_code = var_index;
         break;