Skip to content

Commit

Permalink
Merge pull request #1919 from dtcenter/bugfix_1910_develop_ensemble_s…
Browse files Browse the repository at this point in the history
…tat_lev_in_fcst

Bugfix #1910 develop EnsembleStat level in forecast input template
  • Loading branch information
hankenstein2 authored Nov 11, 2022
2 parents 5ed10ae + f67106f commit 4f1d6e6
Show file tree
Hide file tree
Showing 17 changed files with 163 additions and 144 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import datetime

from metplus.wrappers.command_builder import CommandBuilder
from metplus.util import ti_calculate
from metplus.util import ti_calculate, add_field_info_to_time_info


def get_data_dir(config):
Expand All @@ -27,9 +27,9 @@ def test_find_data_no_dated(metplus_config, data_type):
config = metplus_config

pcw = CommandBuilder(config)
v = {}
v['fcst_level'] = "6"
v['obs_level'] = "6"
var_info = {}
var_info['fcst_level'] = "6"
var_info['obs_level'] = "6"
task_info = {}
task_info['valid'] = datetime.datetime.strptime("201802010000",'%Y%m%d%H%M')
task_info['lead'] = 0
Expand All @@ -39,7 +39,8 @@ def test_find_data_no_dated(metplus_config, data_type):
pcw.c_dict[f'{data_type}FILE_WINDOW_END'] = 3600
pcw.c_dict[f'{data_type}INPUT_DIR'] = get_data_dir(pcw.config)
pcw.c_dict[f'{data_type}INPUT_TEMPLATE'] = "{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}"
obs_file = pcw.find_data(time_info, v, data_type)
add_field_info_to_time_info(time_info, var_info)
obs_file = pcw.find_data(time_info, data_type)
assert obs_file == pcw.c_dict[f'{data_type}INPUT_DIR']+'/20180201_0045'


Expand Down Expand Up @@ -67,7 +68,7 @@ def test_find_data_not_a_path(metplus_config, data_type):
pcw.c_dict[f'{data_type}FILE_WINDOW_END'] = 0
pcw.c_dict[f'{data_type}INPUT_DIR'] = ''
pcw.c_dict[f'{data_type}INPUT_TEMPLATE'] = 'G003'
obs_file = pcw.find_data(time_info, var_info=None, data_type=data_type)
obs_file = pcw.find_data(time_info, data_type=data_type)
assert obs_file == 'G003'


Expand All @@ -76,8 +77,8 @@ def test_find_obs_no_dated(metplus_config):
config = metplus_config

pcw = CommandBuilder(config)
v = {}
v['obs_level'] = "6"
var_info = {}
var_info['obs_level'] = "6"
task_info = {}
task_info['valid'] = datetime.datetime.strptime("201802010000", '%Y%m%d%H%M')
task_info['lead'] = 0
Expand All @@ -87,7 +88,8 @@ def test_find_obs_no_dated(metplus_config):
pcw.c_dict['OBS_FILE_WINDOW_END'] = 3600
pcw.c_dict['OBS_INPUT_DIR'] = get_data_dir(pcw.config)
pcw.c_dict['OBS_INPUT_TEMPLATE'] = "{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}"
obs_file = pcw.find_obs(time_info, v)
add_field_info_to_time_info(time_info, var_info)
obs_file = pcw.find_obs(time_info)
assert obs_file == pcw.c_dict['OBS_INPUT_DIR'] + '/20180201_0045'


Expand All @@ -96,8 +98,8 @@ def test_find_obs_dated(metplus_config):
config = metplus_config

pcw = CommandBuilder(config)
v = {}
v['obs_level'] = "6"
var_info = {}
var_info['obs_level'] = "6"
task_info = {}
task_info['valid'] = datetime.datetime.strptime("201802010000", '%Y%m%d%H%M')
task_info['lead'] = 0
Expand All @@ -107,7 +109,8 @@ def test_find_obs_dated(metplus_config):
pcw.c_dict['OBS_FILE_WINDOW_END'] = 3600
pcw.c_dict['OBS_INPUT_DIR'] = get_data_dir(pcw.config)
pcw.c_dict['OBS_INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d}/{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}'
obs_file = pcw.find_obs(time_info, v)
add_field_info_to_time_info(time_info, var_info)
obs_file = pcw.find_obs(time_info)
assert obs_file == pcw.c_dict['OBS_INPUT_DIR']+'/20180201/20180201_0013'


Expand All @@ -126,8 +129,8 @@ def test_find_obs_offset(metplus_config, offsets, expected_file, offset_seconds)
config = metplus_config

pcw = CommandBuilder(config)
v = {}
v['obs_level'] = "6"
var_info = {}
var_info['obs_level'] = "6"
task_info = {}
task_info['valid'] = datetime.datetime.strptime("2020020112", '%Y%m%d%H')
task_info['lead'] = 0
Expand All @@ -136,7 +139,8 @@ def test_find_obs_offset(metplus_config, offsets, expected_file, offset_seconds)
pcw.c_dict['OFFSETS'] = offsets
pcw.c_dict['OBS_INPUT_DIR'] = get_data_dir(pcw.config)
pcw.c_dict['OBS_INPUT_TEMPLATE'] = "{da_init?fmt=%2H}z.prepbufr.tm{offset?fmt=%2H}.{da_init?fmt=%Y%m%d}"
obs_file, time_info = pcw.find_obs_offset(time_info, v)
add_field_info_to_time_info(time_info, var_info)
obs_file, time_info = pcw.find_obs_offset(time_info)

print(f"OBSFILE: {obs_file}")
print(f"EXPECTED FILE: {expected_file}")
Expand All @@ -153,8 +157,8 @@ def test_find_obs_dated_previous_day(metplus_config):
config = metplus_config

pcw = CommandBuilder(config)
v = {}
v['obs_level'] = "6"
var_info = {}
var_info['obs_level'] = "6"
task_info = {}
task_info['valid'] = datetime.datetime.strptime("201802010000", '%Y%m%d%H%M')
task_info['lead'] = 0
Expand All @@ -164,7 +168,8 @@ def test_find_obs_dated_previous_day(metplus_config):
pcw.c_dict['OBS_INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d}/{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}'
pcw.c_dict['OBS_FILE_WINDOW_BEGIN'] = -3600
pcw.c_dict['OBS_FILE_WINDOW_END'] = 0
obs_file = pcw.find_obs(time_info, v)
add_field_info_to_time_info(time_info, var_info)
obs_file = pcw.find_obs(time_info)
assert obs_file == pcw.c_dict['OBS_INPUT_DIR']+'/20180131/20180131_2345'


Expand All @@ -173,8 +178,9 @@ def test_find_obs_dated_next_day(metplus_config):
config = metplus_config

pcw = CommandBuilder(config)
v = {}
v['obs_level'] = "6"
var_info = {
'obs_level': "6"
}
task_info = {}
task_info['valid'] = datetime.datetime.strptime("201802012345", '%Y%m%d%H%M')
task_info['lead'] = 0
Expand All @@ -184,7 +190,8 @@ def test_find_obs_dated_next_day(metplus_config):
pcw.c_dict['OBS_INPUT_TEMPLATE'] = '{valid?fmt=%Y%m%d}/{valid?fmt=%Y%m%d}_{valid?fmt=%H%M}'
pcw.c_dict['OBS_FILE_WINDOW_BEGIN'] = 0
pcw.c_dict['OBS_FILE_WINDOW_END'] = 3600
obs_file = pcw.find_obs(time_info, v)
add_field_info_to_time_info(time_info, var_info)
obs_file = pcw.find_obs(time_info)
assert obs_file == pcw.c_dict['OBS_INPUT_DIR']+'/20180202/20180202_0013'


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,49 @@ def set_minimum_config_settings(config, set_fields=True):
config.set('config', 'OBS_VAR1_LEVELS', obs_level)


@pytest.mark.parametrize(
'config_overrides, expected_filename', [
# 0 - set forecast level
({'FCST_VAR1_NAME': 'fcst_file',
'FCST_VAR1_LEVELS': 'A06',
'OBS_VAR1_NAME': 'obs_file',
'OBS_VAR1_LEVELS': 'A06',
'FCST_ENSEMBLE_STAT_INPUT_TEMPLATE': '{fcst_name}_A{level?fmt=%3H}',
},
f'{fcst_dir}/fcst_file_A006'),
# 1 - don't set forecast level
({'FCST_ENSEMBLE_STAT_INPUT_TEMPLATE': 'fcst_file_A{level?fmt=%3H}'},
f'{fcst_dir}/fcst_file_A000'),
]
)
@pytest.mark.wrapper_c
def test_ensemble_stat_level_in_template(metplus_config, config_overrides,
expected_filename):

config = metplus_config

set_minimum_config_settings(config, set_fields=False)

# set config variable overrides
for key, value in config_overrides.items():
config.set('config', key, value)

wrapper = EnsembleStatWrapper(config)
assert wrapper.isOK

file_list_dir = wrapper.config.getdir('FILE_LISTS_DIR')
file_list_file = f"{file_list_dir}/20050807000000_12_ensemble_stat.txt"
if os.path.exists(file_list_file):
os.remove(file_list_file)

wrapper.run_all_times()
assert os.path.exists(file_list_file)
with open(file_list_file, 'r') as file_handle:
filenames = file_handle.read().splitlines()[1:]
assert len(filenames) == 1
assert filenames[0] == expected_filename


@pytest.mark.parametrize(
'config_overrides, env_var_values', [
# 0 : no ens, 1 fcst, 1 obs
Expand Down Expand Up @@ -577,8 +620,7 @@ def test_ensemble_stat_single_field(metplus_config, config_overrides,

app_path = os.path.join(config.getdir('MET_BIN_DIR'), wrapper.app_name)
verbosity = f"-v {wrapper.c_dict['VERBOSITY']}"
file_list_dir = os.path.join(wrapper.config.getdir('STAGING_DIR'),
'file_lists')
file_list_dir = wrapper.config.getdir('FILE_LISTS_DIR')
config_file = wrapper.c_dict.get('CONFIG_FILE')
out_dir = wrapper.c_dict.get('OUTPUT_DIR')
expected_cmds = [(f"{app_path} {verbosity} "
Expand Down Expand Up @@ -655,8 +697,7 @@ def test_ensemble_stat_fill_missing(metplus_config, config_overrides,

wrapper = EnsembleStatWrapper(config)

file_list_file = os.path.join(wrapper.config.getdir('STAGING_DIR'),
'file_lists',
file_list_file = os.path.join(wrapper.config.getdir('FILE_LISTS_DIR'),
'20050807000000_12_ensemble_stat.txt')
if os.path.exists(file_list_file):
os.remove(file_list_file)
Expand Down
3 changes: 3 additions & 0 deletions metplus/util/string_manip.py
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,9 @@ def format_thresh(thresh_str):
@returns string of comma-separated list of the threshold(s) with
letter format, i.e. gt3,le5.5,eq7
"""
if isinstance(thresh_str, list):
return format_thresh(','.join(thresh_str))

formatted_thresh_list = []
# separate thresholds by comma and strip off whitespace around values
thresh_list = [thresh.strip() for thresh in thresh_str.split(',')]
Expand Down
42 changes: 27 additions & 15 deletions metplus/util/time_util.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@
from dateutil.relativedelta import relativedelta
import re

from .string_manip import split_level, format_thresh

'''!@namespace TimeInfo
@brief Utility to handle timing in METplus wrappers
@code{.sh}
Expand Down Expand Up @@ -332,20 +334,10 @@ def _format_time_list(string_value, get_met_format, sort_list=True):


def ti_calculate(input_dict_preserve):
out_dict = {}
# copy input dictionary so valid or init can be removed to recalculate it
# without modifying the input to the function
input_dict = input_dict_preserve.copy()

KEYS_TO_COPY = ['custom', 'instance']

# set output dictionary to input items
if 'now' in input_dict.keys():
out_dict['now'] = input_dict['now']
out_dict['today'] = out_dict['now'].strftime('%Y%m%d')

# copy over values of some keys if it is set in input dictionary
for key in KEYS_TO_COPY:
if key in input_dict.keys():
out_dict[key] = input_dict[key]
out_dict = input_dict

# read in input dictionary items and compute missing items
# valid inputs: valid, init, lead, offset
Expand Down Expand Up @@ -381,7 +373,6 @@ def ti_calculate(input_dict_preserve):
else:
out_dict['lead'] = relativedelta(seconds=0)


# set offset to 0 if not specified
if 'offset_hours' in input_dict.keys():
out_dict['offset'] = datetime.timedelta(hours=input_dict['offset_hours'])
Expand All @@ -390,7 +381,6 @@ def ti_calculate(input_dict_preserve):
else:
out_dict['offset'] = datetime.timedelta(seconds=0)


# if init and valid are set, check which was set first via loop_by
# remove the other to recalculate
if 'init' in input_dict.keys() and 'valid' in input_dict.keys():
Expand Down Expand Up @@ -509,3 +499,25 @@ def add_to_time_input(time_input, clock_time=None, instance=None, custom=None):
# otherwise leave it unset so it can be set within the wrapper
if custom:
time_input['custom'] = custom


def add_field_info_to_time_info(time_info, var_info):
"""!Add field information from var_info to the time_info dictionary to use
in string template substitution. Sets new items in time_info.
@param time_info dictionary containing time information to substitute
filename template tags
@param var_info dictionary containing information for the fields to process
"""
if var_info is None:
return

for key, value in var_info.items():
# skip index and extra field info
if key == 'index' or key.endswith('extra'):
continue

if key.endswith('thresh'):
value = format_thresh(value)

time_info[key] = value
2 changes: 1 addition & 1 deletion metplus/wrappers/ascii2nc_wrapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ def find_input_files(self, time_info):
return self.infiles

# get list of files even if only one is found (return_list=True)
obs_path = self.find_obs(time_info, var_info=None, return_list=True)
obs_path = self.find_obs(time_info, return_list=True)
if obs_path is None:
return None

Expand Down
Loading

0 comments on commit 4f1d6e6

Please sign in to comment.