From 3ef0fa395ad99cf30dab51052bba6391d374afc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ga=C3=ABlle=20Rigoudy?= Date: Thu, 2 Jun 2022 11:43:18 +0200 Subject: [PATCH] Bugfixes. --- dr2xml/dr_interface/CMIP6.py | 21 ++++++++++++------- dr2xml/grids.py | 8 +++---- dr2xml/infos.py | 16 +++++++++----- dr2xml/pingfiles_interface.py | 8 +++---- dr2xml/projects/CMIP6.py | 6 ++++++ .../projects_interface_definitions.py | 2 ++ dr2xml/vars_interface/definitions.py | 6 +++++- dr2xml/vars_interface/extra.py | 2 +- dr2xml/vars_interface/generic.py | 2 +- dr2xml/vars_interface/generic_data_request.py | 2 +- dr2xml/vars_interface/selection.py | 3 ++- 11 files changed, 50 insertions(+), 26 deletions(-) diff --git a/dr2xml/dr_interface/CMIP6.py b/dr2xml/dr_interface/CMIP6.py index 1d0e883..ee384b8 100644 --- a/dr2xml/dr_interface/CMIP6.py +++ b/dr2xml/dr_interface/CMIP6.py @@ -95,15 +95,15 @@ def get_sectors_list(): def get_element_uid(id=None, error_msg=None, raise_on_error=False, check_print_DR_errors=True, - check_print_stdnames_error=False): + check_print_stdnames_error=False, elt_type=None): """ Get the uid of an element if precised, else the list of all elements. """ logger = get_logger() if id is None: - return dq.inx.uid + rep = dq.inx.uid elif id in dq.inx.uid: - return dq.inx.uid[id] + rep = dq.inx.uid[id] else: if error_msg is None: error_msg = "DR Error: issue with %s" % id @@ -111,12 +111,15 @@ def get_element_uid(id=None, error_msg=None, raise_on_error=False, check_print_D raise Dr2xmlError(error_msg) elif check_print_DR_errors and print_DR_errors: logger.error(error_msg) - return None elif check_print_stdnames_error and print_DR_stdname_errors: logger.error(error_msg) - return None - else: - return None + rep = None + if rep is not None: + if elt_type in ["variable", ]: + pass + elif elt_type in ["dim", ]: + correct_data_request_dim(rep) + return rep def get_experiment_label(experiment): @@ -154,7 +157,9 @@ def correct_data_request_dim(dim): if dim.label in ["misrBands", ]: dim.dimsize = 16 if dim.type in ["character", ]: - dim.altLabel = "sector" + dim.name = "sector" + else: + dim.name = dim.altLabel # The latter is a bug in DR01.00.21 : typewetla has no value there if dim.label in ["typewetla", ]: dim.value = "wetland" diff --git a/dr2xml/grids.py b/dr2xml/grids.py index 095a041..315f2c3 100644 --- a/dr2xml/grids.py +++ b/dr2xml/grids.py @@ -275,7 +275,7 @@ def change_axes_in_grid(grid_id): # Definitely don't want to change an unnamed axis. Such an axis is # generated by vertical interpolation if not any([ssub.tag in ['interpolate_axis', ] for ssub in sub]): - logger.warning("Cannot normalize an axis in grid %s : no axis_ref for axis %s" % (grid_id, sub)) + logger.warning("Cannot normalize an axis in grid %s: no axis_ref for axis %s" % (grid_id, sub)) else: axis_ref = sub.attrib['axis_ref'] # Just quit if axis doesn't have to be processed @@ -290,7 +290,7 @@ def change_axes_in_grid(grid_id): dim_id = 'dim:{}'.format(dr_axis_id) # print "in change_axis for %s %s"%(grid_id,dim_id) # dim_id should be a dimension ! - dim = get_element_uid(dim_id, + dim = get_element_uid(dim_id, elt_type="dim", error_msg="Value %s in 'non_standard_axes' is not a DR dimension id" % dr_axis_id) # We don't process scalars here if dim.value in ['', ] or dim.label in ["scatratio", ]: @@ -318,7 +318,7 @@ def create_axis_from_dim(dim, labels, axis_ref): constructs generating CMIP6 requested attributes """ axis_id = "DR_" + dim.label + "_" + axis_ref - axis_name = dim.altLabel + axis_name = dim.name if axis_id in get_config_variable("axis_defs"): return axis_id, axis_name # @@ -375,7 +375,7 @@ def scalar_vertical_dimension(sv): Return the altLabel attribute if it is a vertical dimension, else None. """ if 'cids' in sv.struct.__dict__: - cid = get_element_uid(sv.struct.cids[0]) + cid = get_element_uid(sv.struct.cids[0], elt_type="dim") if is_vert_dim(cid): return cid.altLabel return None diff --git a/dr2xml/infos.py b/dr2xml/infos.py index 89bd63a..4d97333 100644 --- a/dr2xml/infos.py +++ b/dr2xml/infos.py @@ -6,7 +6,7 @@ """ from __future__ import print_function, division, absolute_import, unicode_literals -from collections import OrderedDict +from collections import OrderedDict, defaultdict from logger import get_logger @@ -113,12 +113,14 @@ def print_some_stats(context, svars_per_table, skipped_vars_per_table, actually_ if extended: logger.info("\n\nSome Statistics on actually written variables per variable...") dic = OrderedDict() - dic_ln = OrderedDict() + dic_ln = defaultdict(set) for label, long_name, table, frequency, Priority, spatial_shp in actually_written_vars: + dic_ln[label].add(long_name) if label not in dic: - dic[label] = [] - dic_ln.update({label: long_name}) + dic[label] = list() dic[label].append(frequency + '_' + table + '_' + spatial_shp + '_' + str(Priority)) + for label in dic_ln: + dic_ln[label] = sorted(list(dic_ln[label])) list_labels = list(dic) list_labels.sort() @@ -127,9 +129,13 @@ def print_some_stats(context, svars_per_table, skipped_vars_per_table, actually_ for label in list_labels: logger.info((14 + len(label)) * "-") - logger.info("--- VARNAME: %s: %s" % (label, dic_ln[label])) + logger.info("--- VARNAME: %s: %s" % (label, dic_ln[label][0])) logger.info((14 + len(label)) * "-") for val in dic[label]: logger.info(14 * " " + "* %20s %s" % (val, label)) + if len(dic_ln[label]) > 1: + logger.warning(14 * " " + "Warning: several long names are available:") + for long_name in dic_ln[label]: + logger.warning(18 * " " + "- %s" % long_name) return True diff --git a/dr2xml/pingfiles_interface.py b/dr2xml/pingfiles_interface.py index 6a56006..f22adfe 100644 --- a/dr2xml/pingfiles_interface.py +++ b/dr2xml/pingfiles_interface.py @@ -310,8 +310,8 @@ def check_for_file_input(sv, hgrid): remap_grid_def = DR2XMLElement(tag="grid", id=file_grid_id) remap_grid_def.append(DR2XMLElement(tag="domain", domain_ref=file_domain_id)) add_value_in_dict_config_variable(variable="grid_defs", key=file_grid_id, value=remap_grid_def) - logger.info(domain_def) - logger.info(remap_grid_def) + logger.debug(domain_def) + logger.debug(remap_grid_def) # Create xml for reading the variable filename = externs[sv.label][hgrid][get_grid_choice()] @@ -322,7 +322,7 @@ def check_for_file_input(sv, hgrid): file_def.append(DR2XMLElement(tag="field", id=field_in_file_id, name=sv.label, operation="instant", freq_op="1ts", freq_offset="1ts", grid_ref=file_grid_id)) add_value_in_dict_config_variable(variable="file_defs", key=file_id, value=file_def) - logger.info(file_def) + logger.debug(file_def) # # field_def='%s'%\ field_def = DR2XMLElement(tag="field", id=pingvar, grid_ref=grid_id, field_ref=field_in_file_id, @@ -331,4 +331,4 @@ def check_for_file_input(sv, hgrid): context_index = get_config_variable("context_index", to_change=True) context_index[pingvar] = field_def - logger.info(field_def) + logger.debug(field_def) diff --git a/dr2xml/projects/CMIP6.py b/dr2xml/projects/CMIP6.py index f63acaa..501b700 100644 --- a/dr2xml/projects/CMIP6.py +++ b/dr2xml/projects/CMIP6.py @@ -379,6 +379,12 @@ def fill_license(value, institution_id, info_url): ) ) ] + ), + realm=ParameterSettings( + key="realm", + corrections=dict( + ocnBgChem="ocnBgchem" + ) ) ) ) diff --git a/dr2xml/projects/projects_interface_definitions.py b/dr2xml/projects/projects_interface_definitions.py index 5aa1599..5aafb7d 100644 --- a/dr2xml/projects/projects_interface_definitions.py +++ b/dr2xml/projects/projects_interface_definitions.py @@ -450,6 +450,8 @@ def check(self, common_dict=dict(), internal_dict=dict(), additional_dict=dict() additional_dict=additional_dict, allow_additional_keytypes=allow_additional_keytypes) if relevant: + if isinstance(check_value, list) and len(check_value) == 1: + check_value = check_value[0] reference_values = [return_value(reference_value, common_dict=common_dict, internal_dict=internal_dict, additional_dict=additional_dict, allow_additional_keytypes=allow_additional_keytypes) diff --git a/dr2xml/vars_interface/definitions.py b/dr2xml/vars_interface/definitions.py index 80d6e80..6dd02ce 100644 --- a/dr2xml/vars_interface/definitions.py +++ b/dr2xml/vars_interface/definitions.py @@ -108,8 +108,9 @@ class SimpleDim(SimpleObject): """ def __init__(self, label=False, zoom_label=False, stdname=False, long_name=False, positive=False, requested="", value=False, out_name=False, units=False, is_zoom_of=False, bounds=False, boundsValues=False, - axis=False, type=False, coords=False, title=False, is_union_for=list(), **kwargs): + axis=False, type=False, coords=False, title=False, name=None, is_union_for=list(), **kwargs): self.label = label + self.altLabel = self.label self.zoom_label = zoom_label self.stdname = stdname self.long_name = long_name @@ -126,6 +127,7 @@ def __init__(self, label=False, zoom_label=False, stdname=False, long_name=False self.coords = coords self.title = title self.is_union_for = is_union_for + self.name = name super(SimpleDim, self).__init__(**kwargs) def correct_data_request(self): @@ -134,3 +136,5 @@ def correct_data_request(self): self.dimsize = max(len(self.requested.split(" ")), 1) else: self.dimsize = 1 + if self.altLabel != self.label: + self.altLabel = self.label diff --git a/dr2xml/vars_interface/extra.py b/dr2xml/vars_interface/extra.py index 20d53a4..caba1bb 100644 --- a/dr2xml/vars_interface/extra.py +++ b/dr2xml/vars_interface/extra.py @@ -203,7 +203,7 @@ def read_extra_table(path, table): realms=["atmos", "aerosol", "atmosChem"])) extra_var.set_attributes(ref_var=extra_var.label_without_psuffix) extravars.append(extra_var) - logger.info("For extra table %s (which has %d variables): " % (table, len(extravars))) + logger.info("For extra table %s (which has %d variables):" % (table, len(extravars))) logger.info("\tVariables which dim was found in extra coordinates table:\n%s" % "\n".join(["\t\t%20s: %s\n" % (d, " ".join(sorted(dim_from_extra[d]))) for d in sorted(list(dim_from_extra))])) diff --git a/dr2xml/vars_interface/generic.py b/dr2xml/vars_interface/generic.py index 4876d75..c47e0d6 100644 --- a/dr2xml/vars_interface/generic.py +++ b/dr2xml/vars_interface/generic.py @@ -132,7 +132,7 @@ def get_simple_dim_from_dim_id(dimid): """ d = get_element_uid(dimid) # - stdname = get_element_uid(d.standardName, check_print_stdnames_error=True, + stdname = get_element_uid(d.standardName, check_print_stdnames_error=True, check_print_DR_errors=False, error_msg="Issue with standardname for dimid %s" % dimid) if stdname is not None: stdname = stdname.uid diff --git a/dr2xml/vars_interface/generic_data_request.py b/dr2xml/vars_interface/generic_data_request.py index 7c9f43f..252d928 100644 --- a/dr2xml/vars_interface/generic_data_request.py +++ b/dr2xml/vars_interface/generic_data_request.py @@ -213,7 +213,7 @@ def select_data_request_CMORvars_for_lab(sset=False, year=None): % (multiple_grids[-1], repr(d[v]))) if not print_multiple_grids and multiple_grids is not None and len(multiple_grids) > 0: logger.info("\tThese variables will be processed with multiple grids " - "(rerun with print_multiple_grids set to True for details): %s" % repr(multiple_grids.sort())) + "(rerun with print_multiple_grids set to True for details): %s" % repr(sorted(multiple_grids))) # # Print a count of distinct var labels logger.info('Number of distinct var labels is: %d' % len(set([get_element_uid(v).label for v in d]))) diff --git a/dr2xml/vars_interface/selection.py b/dr2xml/vars_interface/selection.py index 64a9397..c58afe8 100644 --- a/dr2xml/vars_interface/selection.py +++ b/dr2xml/vars_interface/selection.py @@ -34,6 +34,7 @@ def check_exclusion(var, *exclusions): reasons = ". ".join(reasons).strip() if len(reasons) == 0: reasons = "Unknown reason." + reasons = " " + reasons return tests, reasons @@ -90,7 +91,7 @@ def select_variables_to_be_processed(year, context, select): else: svars_per_table[svar.mipTable].append(svar) if len(excluded_vars) > 0: - logger.info("The following pairs (variable,table) have been excluded for these reasons :\n%s" % + logger.info("The following pairs (variable,table) have been excluded for these reasons:\n%s" % "\n".join(["%s: %s" % (reason, print_struct(excluded_vars[reason], skip_sep=True, sort=True)) for reason in sorted(list(excluded_vars))])) for table in sorted(list(svars_per_table)):