feature 1608 METDbLoad read MODE/MTD txt files (#1628)

internal_tests/pytests/grid_stat/test_grid_stat_wrapper.py

@@ -1,18 +1,10 @@
 #!/usr/bin/env python3
 
-import os
-import sys
-import re
-import logging
-from collections import namedtuple
 import pytest
+import os
 from datetime import datetime
 
-import produtil
-
 from metplus.wrappers.grid_stat_wrapper import GridStatWrapper
-from metplus.util import met_util as util
-from metplus.util import time_util
 
 fcst_dir = '/some/path/fcst'
 obs_dir = '/some/path/obs'

internal_tests/pytests/met_db_load/test_met_db_load.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+
+import pytest
+
+from metplus.wrappers.met_db_load_wrapper import METDbLoadWrapper
+
+@pytest.mark.parametrize(
+    'filename, expected_result', [
+        ('myfile.png', False),
+        ('anotherfile.txt', False),
+        ('goodfile.stat', True),
+        ('goodfile.tcst', True),
+        ('mode_goodfile.txt', True),
+        ('mtd_goodfile.txt', True),
+        ('monster_badfile.txt', False),
+    ]
+)
+def test_is_loadable_file(filename, expected_result):
+    assert METDbLoadWrapper._is_loadable_file(filename) == expected_result
+
+@pytest.mark.parametrize(
+    'filenames, expected_result', [
+        (['myfile.png',
+          'anotherfile.txt'], False),
+        (['myfile.png',
+          'goodfile.stat'], True),
+        (['myfile.png',
+          'goodfile.tcst',
+          'anotherfile.txt'], True),
+        (['myfile.png',
+          'mode_goodfile.txt'], True),
+        (['myfile.png',
+          'mtd_goodfile.txt'], True),
+        (['myfile.png',
+          'monster_badfile.txt'], False),
+        ([], False),
+    ]
+)
+def test_has_loadable_file(filenames, expected_result):
+    assert METDbLoadWrapper._has_loadable_file(filenames) == expected_result

metplus/wrappers/met_db_load_wrapper.py

@@ -28,21 +28,22 @@ class METDbLoadWrapper(RuntimeFreqWrapper):
          and all c_dict values are prepended with MV_.
          The name is the key and string specifying the type is the value.
     """
-    CONFIG_NAMES = {'HOST': 'string',
-                    'DATABASE': 'string',
-                    'USER': 'string',
-                    'PASSWORD': 'string',
-                    'VERBOSE': 'bool',
-                    'INSERT_SIZE': 'int',
-                    'MODE_HEADER_DB_CHECK': 'bool',
-                    'DROP_INDEXES': 'bool',
-                    'APPLY_INDEXES': 'bool',
-                    'GROUP': 'string',
-                    'LOAD_STAT': 'bool',
-                    'LOAD_MODE': 'bool',
-                    'LOAD_MTD': 'bool',
-                    'LOAD_MPR': 'bool',
-                    }
+    CONFIG_NAMES = {
+        'HOST': 'string',
+        'DATABASE': 'string',
+        'USER': 'string',
+        'PASSWORD': 'string',
+        'VERBOSE': 'bool',
+        'INSERT_SIZE': 'int',
+        'MODE_HEADER_DB_CHECK': 'bool',
+        'DROP_INDEXES': 'bool',
+        'APPLY_INDEXES': 'bool',
+        'GROUP': 'string',
+        'LOAD_STAT': 'bool',
+        'LOAD_MODE': 'bool',
+        'LOAD_MTD': 'bool',
+        'LOAD_MPR': 'bool',
+    }
 
     def __init__(self, config, instance=None):
         met_data_db_dir = config.getdir('MET_DATA_DB_DIR')
@@ -155,7 +156,7 @@ def get_all_files(self, custom=None):
 
     def get_stat_directories(self, input_paths):
         """! Traverse through files under input path and find all directories
-        that contain .stat or .tcst files.
+        that contain .stat, .tcst, mode*.txt, and mtd*.txt files.
 
         @param input_path top level directory to search
         @returns list of unique directories that contain stat files
@@ -165,21 +166,30 @@ def get_stat_directories(self, input_paths):
             self.logger.debug("Finding directories with stat files "
                               f"under {input_path}")
             for root, _, files in os.walk(input_path):
-                for filename in files:
-                    if (not filename.endswith('.stat') and
-                            not filename.endswith('.tcst')):
-                        continue
-                    filepath = os.path.join(root, filename)
-                    stat_dir = os.path.dirname(filepath)
-                    stat_dirs.add(stat_dir)
+                if self._has_loadable_file(files):
+                    stat_dirs.add(root)
 
         stat_dirs = list(stat_dirs)
         for stat_dir in stat_dirs:
             self.logger.info(f"Adding stat file directory: {stat_dir}")
 
         return stat_dirs
 
-    def format_stat_dirs(self, stat_dirs):
+    @staticmethod
+    def _has_loadable_file(files):
+        return any([filename for filename in files
+                    if METDbLoadWrapper._is_loadable_file(filename)])
+
+    @staticmethod
+    def _is_loadable_file(filename):
+        return (filename.endswith('.stat') or
+                filename.endswith('.tcst') or
+                (filename.endswith('.txt') and
+                 (filename.startswith('mode') or
+                  filename.startswith('mtd'))))
+
+    @staticmethod
+    def format_stat_dirs(stat_dirs):
         """! Format list of stat directories to substitute into XML file.
         <vaL></val> tags wil be added around each value.