per #2460, warn instead of error if missing inputs are allowed, track…

… counters for number of runs and missing inputs

internal/tests/pytests/wrappers/grid_diag/test_grid_diag.py

@@ -145,7 +145,7 @@ def test_get_all_files_and_subset(metplus_config, time_info, expected_subset):
                                            filename))
 
     wrapper = GridDiagWrapper(config)
-    assert wrapper.get_all_files()
+    wrapper.c_dict['ALL_FILES'] = wrapper.get_all_files()
 
     # convert list of lists into a single list to compare to expected results
 

internal/tests/pytests/wrappers/series_analysis/test_series_analysis.py

@@ -609,7 +609,7 @@ def test_get_all_files_and_subset(metplus_config, time_info, expect_fcst_subset,
     else:
         wrapper.c_dict['RUN_ONCE_PER_STORM_ID'] = True
 
-    assert wrapper.get_all_files()
+    wrapper.c_dict['ALL_FILES'] = wrapper.get_all_files()
     print(f"ALL FILES: {wrapper.c_dict['ALL_FILES']}")
     expected_fcst = [
         'fcst/20141214_00/ML1201072014/FCST_TILE_F000_gfs_4_20141214_0000_000.nc',

metplus/wrappers/command_builder.py

@@ -1146,7 +1146,6 @@ def check_for_python_embedding(self, input_type, var_info):
         # set file type string to be set in MET config file to specify
         # Python Embedding is being used for this dataset
         file_type = f"file_type = {data_type};"
-        #self.c_dict[f'{input_type}_FILE_TYPE'] = file_type
         self.env_var_dict[f'METPLUS_{input_type}_FILE_TYPE'] = file_type
         return file_ext
 

metplus/wrappers/compare_gridded_wrapper.py

@@ -106,7 +106,9 @@ def run_at_time_once(self, time_info):
                 self.clear()
                 self.c_dict['CURRENT_VAR_INFO'] = var_info
                 add_field_info_to_time_info(time_info, var_info)
+                self.run_count += 1
                 if not self.find_input_files(time_info):
+                    self.missing_input_count += 1
                     continue
                 self.run_at_time_one_field(time_info, var_info)
         else:
@@ -117,7 +119,9 @@ def run_at_time_once(self, time_info):
                 add_field_info_to_time_info(time_info, var_list[0])
 
             self.clear()
+            self.run_count += 1
             if not self.find_input_files(time_info):
+                self.missing_input_count += 1
                 return
             self.run_at_time_all_fields(time_info)
 

metplus/wrappers/extract_tiles_wrapper.py

@@ -209,6 +209,7 @@ def run_at_time_once(self, time_info):
             @param input_dict dictionary containing initialization time
         """
         self.logger.debug("Begin extract tiles")
+        self.run_count += 1
         location_input = self.c_dict.get('LOCATION_INPUT')
         input_path = self.get_location_input_file(time_info, location_input)
         if not input_path:

metplus/wrappers/gen_ens_prod_wrapper.py

@@ -234,7 +234,9 @@ def run_at_time_once(self, time_info):
         if not self.find_field_info(time_info):
             return False
 
+        self.run_count += 1
         if not self.find_input_files(time_info):
+            self.missing_input_count += 1
             return False
 
         if not self.find_and_check_output_file(time_info):

metplus/wrappers/gen_vx_mask_wrapper.py

@@ -146,7 +146,9 @@ def run_at_time_once(self, time_info):
             self.args = do_string_sub(cmd_args,
                                       **time_info)
 
+            self.run_count += 1
             if not self.find_input_files(time_info, temp_file):
+                self.missing_input_count += 1
                 return
 
             # break out of loop if this is the last iteration to

metplus/wrappers/grid_diag_wrapper.py

@@ -143,8 +143,10 @@ def get_command(self):
 
     def run_at_time_once(self, time_info):
         # subset input files as appropriate
+        self.run_count += 1
         input_list_dict = self.subset_input_files(time_info)
         if not input_list_dict:
+            self.missing_input_count += 1
             return
 
         for input_list_file in input_list_dict.values():

metplus/wrappers/mtd_wrapper.py

@@ -154,8 +154,15 @@ def run_at_time_once(self, time_info):
         # get formatted time to use to name file list files
         time_fmt = f"{first_valid_time_info['valid_fmt']}"
 
+        # if no input files were found
+        if not self.c_dict['ALL_FILES']:
+            self.run_count += 1
+            self.missing_input_count += 1
+            return
+
         # loop through the files found for each field (var_info)
         for file_dict in self.c_dict['ALL_FILES']:
+            self.run_count += 1
             var_info = file_dict['var_info']
             inputs = {}
             for data_type in ('FCST', 'OBS'):
@@ -178,12 +185,15 @@ def run_at_time_once(self, time_info):
                 outfile = f"{time_fmt}_mtd_{dt.lower()}_{file_ext}.txt"
                 inputs[data_type] = self.write_list_file(outfile, file_list)
 
-            if not inputs:
-                self.log_error('Input files not found')
-                continue
-            if len(inputs) < 2 and not self.c_dict['SINGLE_RUN']:
-                self.log_error('Could not find all required inputs files')
+            if not inputs or (len(inputs) < 2 and not self.c_dict['SINGLE_RUN']):
+                self.missing_input_count += 1
+                msg = 'Could not find all required inputs files'
+                if self.c_dict['ALLOW_MISSING_INPUTS']:
+                    self.logger.warning(msg)
+                else:
+                    self.log_error(msg)
                 continue
+
             arg_dict = {
                 'obs_path': inputs.get('OBS'),
                 'model_path': inputs.get('FCST'),

metplus/wrappers/pb2nc_wrapper.py

@@ -164,7 +164,6 @@ def find_input_files(self, input_dict):
             @param input_dict dictionary containing some time information
             @returns time info if files are found, None otherwise
         """
-
         infiles, time_info = self.find_obs_offset(input_dict,
                                                   mandatory=True,
                                                   return_list=True)
@@ -195,10 +194,12 @@ def set_valid_window_variables(self, time_info):
     def run_at_time_once(self, input_dict):
         """!Find files needed to run pb2nc and run if found"""
         # look for input files to process
+        self.run_count += 1
         time_info = self.find_input_files(input_dict)
 
         # if no files were found, don't run pb2nc
         if time_info is None:
+            self.missing_input_count += 1
             return
 
         # look for output file path and skip running pb2nc if necessary

metplus/wrappers/pcp_combine_wrapper.py

@@ -495,8 +495,10 @@ def setup_add_method(self, time_info, lookback, data_src):
         # create list of tuples for input levels and optional field names
         self._build_input_accum_list(data_src, time_info)
 
+        self.run_count += 1
         files_found = self.get_accumulation(time_info, lookback, data_src)
         if not files_found:
+            self.missing_input_count += 1
             self.log_error(
                 f'Could not find files to build accumulation in '
                 f"{self.c_dict[f'{data_src}_INPUT_DIR']} using template "
@@ -533,6 +535,7 @@ def setup_derive_method(self, time_info, lookback, data_src):
                                                name=accum_dict['name'],
                                                level=accum_dict['level'],
                                                extra=accum_dict['extra'])
+            self.run_count += 1
             input_files = self.find_data(time_info,
                                          data_type=data_src,
                                          return_list=True)
@@ -547,11 +550,13 @@ def setup_derive_method(self, time_info, lookback, data_src):
                 files_found.append((input_file, field_info))
 
         else:
+            self.run_count += 1
             files_found = self.get_accumulation(time_info,
                                                 lookback,
                                                 data_src,
                                                 field_info_after_file=False)
             if not files_found:
+                self.missing_input_count += 1
                 self.log_error(
                     f'Could not find files to build accumulation in '
                     f"{self.c_dict[f'{data_src}_INPUT_DIR']} using template "

metplus/wrappers/plot_data_plane_wrapper.py

@@ -111,27 +111,13 @@ def get_command(self):
         return cmd
 
     def run_at_time_once(self, time_info):
-        """! Process runtime and try to build command to run ascii2nc
-             Args:
-                @param time_info dictionary containing timing information
-        """
-        self.clear()
-
-        # get input files
-        if not self.find_input_files(time_info):
-            return False
-
-        # get output path
-        if not self.find_and_check_output_file(time_info):
-            return False
-
-        # get other configurations for command
-        self.set_command_line_arguments(time_info)
+        """! Process runtime and try to build command to run plot_data_plane.
+        Calls parent run_at_time_once (RuntimeFreq) then optionally converts
+        PS output to PNG if requested.
 
-        # set environment variables if using config file
-        self.set_environment_variables(time_info)
-
-        if not self.build():
+        @param time_info dictionary containing timing information
+        """
+        if not super().run_at_time_once(time_info):
             return False
 
         if self.c_dict['CONVERT_TO_IMAGE']:

metplus/wrappers/point2grid_wrapper.py

@@ -147,28 +147,6 @@ def get_command(self):
         cmd += ' -v ' + self.c_dict['VERBOSITY']
         return cmd
 
-    def run_at_time_once(self, time_info):
-        """! Process runtime and try to build command to run point2grid
-             Args:
-                @param time_info dictionary containing timing information
-        """
-        # get input files
-        if self.find_input_files(time_info) is None:
-            return
-
-        # get output path
-        if not self.find_and_check_output_file(time_info):
-            return
-
-        # get other configurations for command
-        self.set_command_line_arguments(time_info)
-
-        # set environment variables if using config file
-        self.set_environment_variables(time_info)
-
-        # build command and run
-        self.build()
-
     def find_input_files(self, time_info):
         """!Find input file and mask file and add them to the list of input files.
             Args:

metplus/wrappers/regrid_data_plane_wrapper.py

@@ -319,7 +319,9 @@ def run_at_time_once(self, time_info):
             return False
 
         add_field_info_to_time_info(time_info, var_list[0])
+        self.run_count += 1
         if not self.find_input_files(time_info, data_type):
+            self.missing_input_count += 1
             return False
 
         # set environment variables

metplus/wrappers/runtime_freq_wrapper.py

@@ -199,9 +199,9 @@ def run_once(self, custom):
 
         time_info = time_util.ti_calculate(time_input)
 
-        if not self.get_all_files(custom):
-            self.log_error("A problem occurred trying to obtain input files")
-            return None
+        self.c_dict['ALL_FILES'] = self.get_all_files(custom)
+        if not self._check_input_files():
+            return False
 
         self.clear()
         return self.run_at_time_once(time_info)
@@ -229,6 +229,8 @@ def run_once_per_init_or_valid(self, custom):
             time_info = time_util.ti_calculate(time_input)
 
             self.c_dict['ALL_FILES'] = self.get_all_files_from_leads(time_info)
+            if not self._check_input_files():
+                continue
 
             self.clear()
             if not self.run_at_time_once(time_info):
@@ -260,6 +262,8 @@ def run_once_per_lead(self, custom):
             time_info = time_util.ti_calculate(time_input)
 
             self.c_dict['ALL_FILES'] = self.get_all_files_for_lead(time_info)
+            if not self._check_input_files():
+                continue
 
             self.clear()
             if not self.run_at_time_once(time_info):
@@ -313,6 +317,8 @@ def run_at_time(self, input_dict):
             all_files = []
             self._update_list_with_new_files(time_info, all_files)
             self.c_dict['ALL_FILES'] = all_files
+            if not self._check_input_files():
+                continue
 
             # Run for given init/valid time and forecast lead combination
             self.clear()
@@ -332,7 +338,9 @@ def run_at_time_once(self, time_info):
          False if something went wrong
         """
         # get input files
+        self.run_count += 1
         if not self.find_input_files(time_info):
+            self.missing_input_count += 1
             return False
 
         # get output path
@@ -363,7 +371,7 @@ def get_all_files(self, custom=None):
         # loop over all init/valid times
         for time_input in time_generator(self.config):
             if time_input is None:
-                return False
+                return []
 
             add_to_time_input(time_input,
                               instance=self.instance,
@@ -372,10 +380,18 @@ def get_all_files(self, custom=None):
             lead_files = self.get_all_files_from_leads(time_input)
             all_files.extend(lead_files)
 
-        if not all_files:
-            return False
+        return all_files
 
-        self.c_dict['ALL_FILES'] = all_files
+    def _check_input_files(self):
+        self.run_count += 1
+        if not self.c_dict['ALL_FILES'] and self.app_name != 'user_script':
+            self.missing_input_count += 1
+            msg = 'A problem occurred trying to obtain input files'
+            if self.c_dict['ALLOW_MISSING_INPUTS']:
+                self.logger.warning(msg)
+            else:
+                self.log_error(msg)
+            return False
         return True
 
     def get_all_files_from_leads(self, time_input):
@@ -535,7 +551,7 @@ def subset_input_files(self, time_info, output_dir=None, leads=None,
                or None if could not find any files
         """
         all_input_files = {}
-        if not self.c_dict.get('ALL_FILES'):
+        if not self.c_dict.get('ALL_FILES') or self.c_dict.get('ALL_FILES') is True:
             return all_input_files
 
         if leads is None: