Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Feature 1965 ioda2nc same input #2006

Merged
merged 6 commits into from
Jan 12, 2022
Merged

Feature 1965 ioda2nc same input #2006

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0273cf1
#1965 Excludes the duplicated variable names from multiple input files
Jan 7, 2022
ab1199b
#1965 Processed with the raw variable names and update them when the …
Jan 11, 2022
d5cc7b1
#1965 Added unit test ioda2nc_same_input
Jan 11, 2022
e48c5ff
#1965 Processed with the raw variable names and update them when the …
Jan 11, 2022
5617aa2
#1965 Added unit test ioda2nc_same_input
Jan 11, 2022
ee76143
#1965 initialize var_idx
hsoh-u Jan 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 43 additions & 34 deletions met/src/tools/other/ioda2nc/ioda2nc.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -369,7 +369,8 @@ void process_ioda_file(int i_pb) {
IntArray diff_file_times;
int diff_file_time_count;
StringArray variables_big_nlevels;
static const char *method_name = "process_ioda_file() ";
static const char *method_name = "process_ioda_file() ->";
static const char *method_name_s = "process_ioda_file() ";

bool apply_grid_mask = (conf_info.grid_mask.nx() > 0 &&
conf_info.grid_mask.ny() > 0);
Expand All @@ -388,7 +389,7 @@ void process_ioda_file(int i_pb) {

// Check for a valid file
if(IS_INVALID_NC_P(f_in)) {
mlog << Error << "\n" << method_name << "-> "
mlog << Error << "\n" << method_name
<< "can't open input NetCDF file \"" << ioda_files[i_pb]
<< "\" for reading.\n\n";
delete f_in;
Expand Down Expand Up @@ -444,7 +445,7 @@ void process_ioda_file(int i_pb) {
dim_names, metadata_vars);

if(!is_netcdf_ready) {
mlog << Error << "\n" << method_name << "-> "
mlog << Error << "\n" << method_name
<< "Please check the IODA file (required dimensions or meta variables are missing).\n\n";
delete f_in;
f_in = (NcFile *) 0;
Expand All @@ -460,7 +461,7 @@ void process_ioda_file(int i_pb) {
if(dim_names.has("nrecs")) nrecs = get_dim_value(f_in, "nrecs", false);
else {
nrecs = nvars * nlocs;
mlog << Debug(3) << "\n" << method_name << "-> "
mlog << Debug(3) << "\n" << method_name
<< "nrecs dimension does not exist, so computed\n";
}
NcVar in_hdr_vld_var = get_var(f_in, "datetime@MetaData");
Expand All @@ -472,10 +473,10 @@ void process_ioda_file(int i_pb) {
else {
NcDim datetime_dim = get_nc_dim(&in_hdr_vld_var, 1);
ndatetime = IS_VALID_NC(datetime_dim) ? get_dim_size(&datetime_dim) : nstring;
mlog << Debug(3) << "\n" << method_name << "-> "
mlog << Debug(3) << "\n" << method_name
<< "ndatetime dimension does not exist!\n";
}
mlog << Debug(5) << method_name << "-> dimensions: nvars=" << nvars << ", nlocs=" << nlocs
mlog << Debug(5) << method_name << "dimensions: nvars=" << nvars << ", nlocs=" << nlocs
<< ", nrecs=" << nrecs << ", nstring=" << nstring << ", ndatetime=" << ndatetime << "\n";

npbmsg_total = npbmsg = nlocs;
Expand Down Expand Up @@ -515,36 +516,34 @@ void process_ioda_file(int i_pb) {
}

if(!get_nc_data(&in_hdr_lat_var, hdr_lat_arr, nlocs)) {
mlog << Error << "\n" << method_name << " -> "
mlog << Error << "\n" << method_name
<< "trouble getting latitude\n\n";
exit(1);
}
if(!get_nc_data(&in_hdr_lon_var, hdr_lon_arr, nlocs)) {
mlog << Error << "\n" << method_name << " -> "
mlog << Error << "\n" << method_name
<< "trouble getting longitude\n\n";
exit(1);
}
if(!get_nc_data(&in_hdr_vld_var, hdr_vld_block, lengths, offsets)) {
mlog << Error << "\n" << method_name << " -> "
mlog << Error << "\n" << method_name
<< "trouble getting datetime\n\n";
exit(1);
}

StringArray raw_var_names;
if(do_all_vars || obs_var_names.n() == 0) raw_var_names = obs_value_vars;
else raw_var_names = obs_var_names;
if(obs_var_names.n() > 0) obs_var_names.clear();

NcVar obs_var, qc_var;
ConcatString unit_attr;
ConcatString desc_attr;
map<ConcatString,ConcatString> name_map = conf_info.getObsVarMap();
for(idx=0; idx<raw_var_names.n(); idx++ ) {
int *qc_data = new int[nlocs];
float *obs_data = new float[nlocs];
ConcatString obs_var_name = raw_var_names[idx] + "@ObsValue";
mlog << Debug(7) << method_name << " -> "

mlog << Debug(7) << method_name
<< "processing \"" << obs_var_name << "\" variable!\n";
obs_var = get_var(f_in, obs_var_name.c_str());
v_qc_data.push_back(qc_data);
Expand All @@ -556,13 +555,15 @@ void process_ioda_file(int i_pb) {
get_var_units(&obs_var, unit_attr);
get_att_value_string(&obs_var, "long_name", desc_attr);
}
obs_var_units.add(unit_attr);
obs_var_descs.add(desc_attr);


// Replace the input variable name to the output variable name
ConcatString new_name = name_map[raw_var_names[idx]];
if (0 < new_name.length()) obs_var_names.add(new_name);
else obs_var_names.add(raw_var_names[idx]);
ConcatString raw_name = raw_var_names[idx];
// Filter out the same variable names from multiple input files
if (!obs_var_names.has(raw_name)) {
obs_var_names.add(raw_name);
obs_var_units.add(unit_attr);
obs_var_descs.add(desc_attr);
}
}

// Initialize counts
Expand All @@ -573,7 +574,7 @@ void process_ioda_file(int i_pb) {
bool showed_progress = false;
if(mlog.verbosity_level() >= debug_level_for_performance) {
end_t = clock();
mlog << Debug(debug_level_for_performance) << " PERF: " << method_name << " "
mlog << Debug(debug_level_for_performance) << " PERF: " << method_name_s << " "
<< (end_t-start_t)/double(CLOCKS_PER_SEC)
<< " seconds for preparing\n";
start_t = clock();
Expand Down Expand Up @@ -612,7 +613,7 @@ void process_ioda_file(int i_pb) {

char valid_time[ndatetime+1];
m_strncpy(valid_time, (const char *)(hdr_vld_block + (i_read * ndatetime)),
ndatetime, method_name, "valid_time", true);
ndatetime, method_name_s, "valid_time", true);
valid_time[ndatetime] = 0;
msg_ut = yyyymmddThhmmss_to_unix(valid_time);

Expand Down Expand Up @@ -662,7 +663,7 @@ void process_ioda_file(int i_pb) {

if(has_msg_type) {
int buf_len = sizeof(modified_hdr_typ);
m_strncpy(hdr_typ, hdr_msg_types+(i_read*nstring), nstring, method_name, "hdr_typ");
m_strncpy(hdr_typ, hdr_msg_types+(i_read*nstring), nstring, method_name_s, "hdr_typ");
m_rstrip(hdr_typ, nstring);

// If the message type is not listed in the configuration
Expand All @@ -675,22 +676,22 @@ void process_ioda_file(int i_pb) {

if(0 < message_type_map.count((string)hdr_typ)) {
ConcatString mappedMessageType = message_type_map[(string)hdr_typ];
mlog << Debug(6) << "\n" << method_name << " -> "
mlog << Debug(6) << "\n" << method_name
<< "Switching report type \"" << hdr_typ
<< "\" to message type \"" << mappedMessageType << "\".\n";
if(mappedMessageType.length() < HEADER_STR_LEN) buf_len = HEADER_STR_LEN;
m_strncpy(modified_hdr_typ, mappedMessageType.c_str(), buf_len,
method_name, "modified_hdr_typ");
method_name_s, "modified_hdr_typ");
}
else {
m_strncpy(modified_hdr_typ, hdr_typ, buf_len, method_name, "modified_hdr_typ2");
m_strncpy(modified_hdr_typ, hdr_typ, buf_len, method_name_s, "modified_hdr_typ2");
}
modified_hdr_typ[buf_len-1] = 0;
}

if(has_station_id) {
char tmp_sid[nstring+1];
m_strncpy(tmp_sid, hdr_station_ids+(i_read*nstring), nstring, method_name, "tmp_sid");
m_strncpy(tmp_sid, hdr_station_ids+(i_read*nstring), nstring, method_name_s, "tmp_sid");
m_rstrip(tmp_sid, nstring);
hdr_sid = tmp_sid;
}
Expand Down Expand Up @@ -775,7 +776,14 @@ void process_ioda_file(int i_pb) {

n_hdr_obs = 0;
for(idx=0; idx<v_obs_data.size(); idx++ ) {
obs_arr[1] = idx;
int var_idx = 0;
if (!obs_var_names.has(raw_var_names[idx], var_idx)) {
mlog << Warning << "\n" << method_name
<< "Skip the variable " << raw_var_names[idx]
<< " at " << ioda_files[i_pb] << "\n\n";
continue;
}
obs_arr[1] = var_idx;
georgemccabe marked this conversation as resolved.
Show resolved Hide resolved
obs_arr[2] = obs_pres_arr[i_read];
obs_arr[3] = obs_hght_arr[i_read];
obs_arr[4] = v_obs_data[idx][i_read];
Expand All @@ -795,8 +803,6 @@ void process_ioda_file(int i_pb) {
// store the header data and increment the IODA record
// counter
if(n_hdr_obs > 0) {
//nc_point_obs.add_header(modified_hdr_typ, hdr_sid.c_str(), hdr_vld_ut,
// hdr_lat, hdr_lon, hdr_elv);
i_msg++;
}
else {
Expand Down Expand Up @@ -839,12 +845,12 @@ void process_ioda_file(int i_pb) {
<< n_file_obs << "\n";

if(npbmsg == rej_vld && 0 < rej_vld) {
mlog << Warning << "\n" << method_name << " -> "
mlog << Warning << "\n" << method_name
<< "All records were filtered out by valid time.\n"
<< "\tPlease adjust time range with \"-valid_beg\" and \"-valid_end\".\n"
<< "\tmin/max obs time from IODA file: " << min_time_str
<< " and " << max_time_str << ".\n"
<< "\ttime range: " << start_time_str << " and " << end_time_str << ".\n";
<< "\ttime range: " << start_time_str << " and " << end_time_str << ".\n\n";
}
else {
mlog << Debug(1) << "Obs time between " << unix_to_yyyymmdd_hhmmss(min_msg_ut)
Expand Down Expand Up @@ -882,13 +888,13 @@ void process_ioda_file(int i_pb) {

if(mlog.verbosity_level() >= debug_level_for_performance) {
method_end = clock();
cout << " PERF: " << method_name << " "
cout << " PERF: " << method_name_s
<< (method_end-method_start)/double(CLOCKS_PER_SEC)
<< " seconds\n";
}

if(i_msg <= 0) {
mlog << Warning << "\n" << method_name << " -> "
mlog << Warning << "\n" << method_name
<< "No IODA records retained from file: "
<< ioda_files[i_pb] << "\n\n";
}
Expand Down Expand Up @@ -922,9 +928,12 @@ void write_netcdf_hdr_data() {
StringArray nc_var_desc_arr;
const long var_count = obs_var_names.n();
const long units_count = obs_var_units.n();
map<ConcatString,ConcatString> name_map = conf_info.getObsVarMap();

for(int i=0; i<var_count; i++) {
nc_var_name_arr.add(obs_var_names[i]);
ConcatString new_name = name_map[obs_var_names[i]];
if (0 >= new_name.length()) new_name = obs_var_names[i];
nc_var_name_arr.add(new_name);
}
for(int i=0; i<units_count; i++) {
nc_var_unit_arr.add(obs_var_units[i]);
Expand Down
20 changes: 20 additions & 0 deletions test/xml/unit_ioda2nc.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,4 +100,24 @@
</output>
</test>

<test name="ioda2nc_same_input">
<exec>&MET_BIN;/ioda2nc</exec>
<env>
<pair><name>STATION_ID</name> <value>"KEKA"</value></pair>
<pair><name>MASK_GRID</name> <value></value></pair>
<pair><name>MASK_POLY</name> <value></value></pair>
<pair><name>MESSAGE_TYPE</name> <value></value></pair>
</env>
<param> \
&DATA_DIR_OBS;/ioda/ioda.NC001007.2020031012.nc \
&OUTPUT_DIR;/ioda2nc/ioda.NC001007.2020031012.same_input.nc \
-config &CONFIG_DIR;/IODA2NCConfig_mask \
-iodafile &DATA_DIR_OBS;/ioda/ioda.NC001007.2020031012.nc \
-v 2
</param>
<output>
<point_nc>&OUTPUT_DIR;/ioda2nc/ioda.NC001007.2020031012.same_input.nc</point_nc>
</output>
</test>

</met_test>