Merge pull request #32 from rigoudyg/V3.1

V3.1

.gitignore

@@ -4,6 +4,8 @@
 /spare.py
 *~
 *.pyc
+__pycache__
+.ipynb_checkpoints
 .ipynb_checkpoints/*
 toy/
 old/
@@ -28,3 +30,8 @@ create_file_defs.sh
 tests/htmlcov
 tests/test*/output_*
 /sphinx/build/
+/dr2xml_list_perso_and_dev_file_names
+dr2xml_output/dr2xml_*.xml
+dr2xml_output/exercices/*/dr2xml_*.xml
+*.pyc
+/scripts/output_labs/

.readthedocs.yaml

@@ -0,0 +1,22 @@
+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Set the version of Python and other tools you might need
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: sphinx/source/conf.py
+
+# We recommend specifying your dependencies to enable reproducible builds:
+# https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html
+python:
+  install:
+  - requirements: requirements.txt

dr2xml/Xwrite.py

@@ -24,7 +24,7 @@
 from .config import get_config_variable, set_config_variable, add_value_in_dict_config_variable
 
 # Interface to Data Request
-from .dr_interface import get_scope
+from .dr_interface import get_dr_object
 
 from .xml_interface import DR2XMLElement, create_pretty_xml_doc, find_rank_xml_subelement, wrv
 
@@ -44,7 +44,7 @@
 
 # Variables tools
 from .vars_interface.cmor import ping_alias, get_simplevar
-from .vars_interface.generic_data_request import endyear_for_CMORvar, get_grid_choice
+from .vars_interface.generic_data_request import endyear_for_CMORvar
 
 # Post-processing tools
 from .postprocessing import process_vertical_interpolation, process_zonal_mean, process_diurnal_cycle, \
@@ -118,17 +118,17 @@ def create_xios_aux_elmts_defs(sv, alias, table, context, target_hgrid_id, zgrid
     # must exclude COSP outputs which are already interpolated to height or P7 levels
     logger.debug("Deal with %s, %s, %s" % (sv.label, prefix + sv.label, sv.label_without_psuffix))
     pingvars = get_config_variable("pingvars")
-    if (ssh[0:4] == 'XY-P' and ssh != 'XY-P7') or \
-            ssh[0:3] == 'Y-P' or (ssh == "XY-perso" and prefix + sv.label not in pingvars) or \
-            ((ssh[0:5] == 'XY-na' or ssh[0:4] == 'Y-na') and
+    if (ssh.startswith('XY-P') and ssh not in ['XY-P7', ]) or \
+            ssh.startswith('Y-P') or (ssh in ["XY-perso", ] and prefix + sv.label not in pingvars) or \
+            ((ssh.startswith('XY-na') or ssh.startswith('Y-na')) and
              prefix + sv.label not in pingvars and sv.label_without_psuffix != sv.label):
         # TBD check - last case is for singleton
         last_grid_id, last_field_id = process_vertical_interpolation(sv, alias, last_grid_id)
         # If vertical interpolation is done, change the value of those boolean to modify the behaviour of dr2xml
         grid_with_vertical_interpolation = True
     elif ssh in ["XY-HG", ]:
         # Handle interpolation on a height level over the ground
-        logger.info("Deal with XY-HG spatial shape for %s,%s" % (sv.label, sv.ref_var))
+        logger.debug("Deal with XY-HG spatial shape for %s,%s" % (sv.label, sv.ref_var))
         last_field_id, last_grid_id = process_levels_over_orog(sv, alias, last_grid_id)
 
     #
@@ -440,10 +440,14 @@ def write_xios_file_def(filename, svars_per_table, year, dummies, skipped_vars_p
             ok = False
             logger.warning("Model component %s is required by CMIP6 CV for experiment %s and not present "
                            "(present=%s)" % (c, experiment_id, repr(actual_components)))
-    if len(allowed_components) > 0:
-        for c in actual_components:
-            if c not in allowed_components and c not in required_components:
-                ok = False or internal_dict['bypass_CV_components']
+    if internal_dict["bypass_CV_components"]:
+        ok = True
+    elif len(allowed_components) > 0:
+        not_allowed_components = [c for c in actual_components if c not in allowed_components and
+                                  c not in required_components]
+        if len(not_allowed_components) > 0:
+            ok = False
+            for c in not_allowed_components:
                 logger.warning("Warning: Model component %s is present but not required nor allowed (%s)" %
                                (c, repr(allowed_components)))
     if not ok:
@@ -458,7 +462,7 @@ def write_xios_file_def(filename, svars_per_table, year, dummies, skipped_vars_p
     set_config_variable("domain_defs", OrderedDict())
     # Add xml_file_definition
     xml_file_definition = DR2XMLElement(tag="file_definition")
-    _, hgrid, _, _, _ = internal_dict['grids'][get_grid_choice()][context]
+    _, hgrid, _, _, _ = internal_dict['grids'][get_settings_values("internal_values", "grid_choice")][context]
     files_list = determine_files_list(svars_per_table, enddate, year, debug)
     for file_dict in files_list:
         write_xios_file_def_for_svars_list(hgrid=hgrid, xml_file_definition=xml_file_definition, dummies=dummies,
@@ -578,8 +582,8 @@ def write_xios_file_def_for_svars_list(vars_list, hgrid, xml_file_definition, fr
                                                    target_hgrid_id=target_hgrid_id, zgrid_id=zgrid_id,
                                                    alias_ping=alias_ping, source_grid=source_grid)
             xml_file.append(end_field)
-            actually_written_vars.append((svar.label, svar.long_name, svar.mipTable, svar.frequency, svar.Priority,
-                                          svar.spatial_shp))
+            actually_written_vars.append((svar.label, svar.long_name, svar.stdname, svar.mipTable, svar.frequency,
+                                          svar.Priority, svar.spatial_shp))
     # Add content to xml_file to out
     if found:
         if found_begin_A:
@@ -763,7 +767,7 @@ def get_split_info(sv, table, enddate, year, debug):
             endmonth = "01"
             endday = "01"
             split_last_date = "{}-{}-{} 00:00:00".format(endyear, endmonth, endday)
-        sc = get_scope()
+        sc = get_dr_object("get_scope")
         split_freq = determine_split_freq(sv, grid_choice, sc.mcfg, context)
     return split_freq_format, split_last_date, split_start_offset, split_end_offset, split_freq
 
@@ -799,7 +803,7 @@ def get_grid_info(sv, grid, table):
             target_hgrid_id = cfsites_domain_id
             zgrid_id = None
         else:
-            target_hgrid_id = internal_dict["ping_variables_prefix"] + grid
+            target_hgrid_id = internal_dict["grid_prefix"] + grid
             zgrid_id = "TBD : Should create zonal grid for CMIP6 standard grid %s" % grid
         grid_label, grid_resolution, grid_description = DR_grid_to_grid_atts(grid)
 

dr2xml/__init__.py

@@ -557,41 +557,45 @@
 }
 
 
-def generate_file_defs(lset, sset, year, enddate, context, cvs_path, pingfiles=None,
-                       dummies='include', printout=False, dirname="./", prefix="", attributes=list(),
-                       select="on_expt_and_year", debug=False):
+def configuration_init(func):
     """
-    A wrapper for profiling top-level function : generate_file_defs_inner
+    This function is a configurator for others entry points into dr2xml.
+    :param func: Function to be decorated
+    :param dict lset: dictionary containing lab and model related settings
+    :param dict sset: dictionary containing simulation related settings
+    :param six.string_types cvs_path: path where controled vocabulary can be found
+    :param bool printout: print infos
+    :param six.string_types prefix: prefix used for each file definition
+    :param bool debug: Turn on debug mode to have more information during the run.
+    :param bool force_reset: Should the internal values be emptied during to related runs on the same python sequence?
+                             If False (default), save time by not reading once again Data Request but can cause filter
+                             issues if some things should be different (not the case for most usages).
+                             Else, the run is a little longer but all changes are taken into account.
+
+    :return: The initial function with initialized environment to use dr2xml.
     """
-    if debug:
-        default_level = "debug"
-    elif printout:
-        default_level = "info"
-    else:
-        default_level = "warning"
-    initialize_logger(default=True, level=default_level)
-    # pr = cProfile.Profile()
-    # pr.enable()
-    # Initialize lset and sset variables for all functions
-    initialize_config_variables()
-    initialize_settings(lset=lset, sset=sset, cvspath=cvs_path, context=context, prefix=prefix,
-                        root=os.path.basename(os.path.abspath(__file__)), year=year)
-    generate_file_defs_inner(year, enddate, context, pingfiles=pingfiles, dummies=dummies, dirname=dirname,
-                             attributes=attributes, select=select)
-    # pr.disable()
-    # if python_version == "python2":
-    #     s = io.BytesIO()
-    # else:
-    #     s = io.StringIO()
-    # sortby = 'cumulative'
-    # ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
-    # ps.print_stats()
-    # Just un-comment next line to get the profile on stdout
-    # print(s.getvalue())
-
-
-def generate_file_defs_inner(year, enddate, context, pingfiles=None, dummies='include', dirname="./", attributes=list(),
-                             select="on_expt_and_year"):
+    def make_configuration(lset, sset, cvs_path=None, printout=False, prefix="", debug=False, force_reset=False,
+                           **kwargs):
+        year = kwargs.get("year", 0)
+        context = kwargs.get("context")
+        dirname = kwargs.get("dirname")
+        if debug:
+            default_level = "debug"
+        elif printout:
+            default_level = "info"
+        else:
+            default_level = "warning"
+        initialize_logger(default=True, level=default_level)
+        initialize_config_variables()
+        initialize_settings(lset=lset, sset=sset, cvspath=cvs_path, context=context, prefix=prefix,
+                            year=year, dirname=dirname, force_reset=force_reset)
+        return func(**kwargs)
+    return make_configuration
+
+
+@configuration_init
+def generate_file_defs(year, enddate, context, pingfiles=None, dummies='include', dirname="./", attributes=list(),
+                       select="on_expt_and_year"):
     """
     Using the DR module, a dict of lab settings ``lset``, and a dict
     of simulation settings ``sset``, generate an XIOS file_defs 'file' for a
@@ -602,18 +606,19 @@ def generate_file_defs_inner(year, enddate, context, pingfiles=None, dummies='in
     correspondance between a context and a few realms
 
 on
-    :param six.string_types context: XIOS context considered for the launch
-es with a different name between model
+    :param six.string_types year: year associated with the launch of dr2xml
+    :param six.string_types enddate: enddate of the current launch of dr2xml
+    :param six.string_types context: XIOS context considered for the launches with a different name between model
                                        and Data Request
+    :param None or list of six.string_types pingfiles: Ping files which define for a given Data Request variable
+                                                       the associated model variable.
     :param six.string_types dummies: specify how to treat dummy variables among:
 
         - "include": include dummy refs in file_def (useful for demonstration run)
         - "skip": don't write field with a ref to a dummy (useful until ping_file is fully completed)
         - "forbid": stop if any dummy (useful for production run)
 
-    :param bool printout: print infos
     :param six.string_types dirname: directory in which outputs will be created
-    :param six.string_types prefix: prefix used for each file definition
     :param list attributes: list of (name,value) pairs which are to be inserted as
                             additional file-level attributes. They are complemented with entry
                             "non_standard__attributes" of dict sset
@@ -627,7 +632,7 @@ def generate_file_defs_inner(year, enddate, context, pingfiles=None, dummies='in
 
     """
     # Interface to Data Request
-    from .dr_interface import get_DR_version
+    from .dr_interface import get_dr_object
 
     # Tools to deal with ping files
     from .pingfiles_interface import read_pingfiles_variables
@@ -638,7 +643,6 @@ def generate_file_defs_inner(year, enddate, context, pingfiles=None, dummies='in
     # Variables tools
     from .vars_interface.generic import multi_plev_suffixes, single_plev_suffixes
     from .vars_interface.selection import select_variables_to_be_processed
-    from .vars_interface.generic_data_request import initialize_sn_issues
 
     # XIOS reading and writing tools
     from .Xparse import init_context
@@ -660,7 +664,7 @@ def generate_file_defs_inner(year, enddate, context, pingfiles=None, dummies='in
     # TBS# from os import path as os_path
     # TBS# prog_path=os_path.abspath(os_path.split(__file__)[0])
 
-    print("* %29s" % "CMIP6 Data Request version: ", get_DR_version())
+    print("* %29s" % "CMIP6 Data Request version: ", get_dr_object("get_data_request").get_version())
     print("\n*\n {}".format(50 * "*"))
 
     logger = get_logger()
@@ -689,7 +693,6 @@ def generate_file_defs_inner(year, enddate, context, pingfiles=None, dummies='in
         sys.exit(1)
     set_config_variable("cell_method_warnings", list())
     warnings_for_optimisation = list()
-    initialize_sn_issues(OrderedDict())
     #
     # --------------------------------------------------------------------
     # Select variables that should be processed
@@ -717,6 +720,7 @@ def generate_file_defs_inner(year, enddate, context, pingfiles=None, dummies='in
     # Write XIOS file_def
     # --------------------------------------------------------------------
     # filename=dirname+"filedefs_%s.xml"%context
+    os.chdir(dirname)
     filename = dirname + "dr2xml_%s.xml" % context
     write_xios_file_def(filename, svars_per_table, year, dummies, skipped_vars_per_table, actually_written_vars,
                         context, enddate, attributes)
@@ -746,3 +750,24 @@ def generate_file_defs_inner(year, enddate, context, pingfiles=None, dummies='in
         for w in warnings_for_optimisation:
             logger.warning(w.replace(internal_settings['ping_variables_prefix'], ""))
         print()
+
+
+@configuration_init
+def create_ping_files(context, path_special, dummy="field_atm", dummy_with_shape=False, exact=False, comments=False,
+                      filename=None, debug=list(), by_realm=False):
+    from .settings_interface import get_settings_values
+    from .vars_interface.selection import select_variables_to_be_processed
+    from .pingfiles_interface import ping_file_for_realms_list
+
+    considered_realms = get_settings_values("internal", "realms_per_context")
+    svars = select_variables_to_be_processed(None, context, "no")
+    if by_realm:
+        for realm in considered_realms:
+            ping_file_for_realms_list(context=context, svars=svars, lrealms=[realm, ], path_special=path_special,
+                                      dummy=dummy, dummy_with_shape=dummy_with_shape, exact=exact, comments=comments,
+                                      debug=debug, filename=filename % realm)
+    else:
+        ping_file_for_realms_list(context=context, svars=svars, lrealms=considered_realms, path_special=path_special,
+                                  dummy=dummy, dummy_with_shape=dummy_with_shape, exact=exact, comments=comments,
+                                  filename=filename, debug=debug)
+