Merge branch 'main' into cdx_GERICS_gp-differences_fix

.zenodo.json

@@ -178,6 +178,11 @@
             "name": "Kazeroni, Rémi",
             "orcid": "0000-0001-7205-9528"
         },
+        {
+            "affiliation": "GEOMAR, Germany",
+            "name": "Hohn, David",
+            "orcid": "0000-0002-5317-1247"
+        },
         {
             "affiliation": "DLR, Germany",
             "name": "Bauer, Julian"
@@ -190,6 +195,11 @@
         {
             "affiliation": "Forschungszentrum Juelich, Germany",
             "name": "Benke, Joerg"
+        },
+        {
+            "affiliation": "BSC, Spain",
+            "name": "Martin-Martinez, Eneko",
+            "orcid": "0000-0002-9213-7818"
         }
     ],
     "description": "ESMValCore: A community tool for pre-processing data from Earth system models in CMIP and running analysis scripts.",

CITATION.cff

@@ -199,6 +199,11 @@ authors:
     affiliation: "Forschungszentrum Juelich (FZJ), Germany"
     family-names: Benke
     given-names: Joerg
+  -
+    affiliation: "BSC, Spain"
+    family-names: Martin-Martinez
+    given-names: Eneko
+    orcid: "https://orcid.org/0000-0002-9213-7818"
 
 cff-version: 1.2.0
 date-released: 2023-12-19

esmvalcore/_main.py

@@ -353,8 +353,10 @@ def run(self,
             Recipe to run, as either the name of an installed recipe or the
             path to a non-installed one.
         config_file: str, optional
-            Configuration file to use. If not provided the file
-            ${HOME}/.esmvaltool/config-user.yml will be used.
+            Configuration file to use. Can be given as absolute or relative
+            path. In the latter case, search in the current working directory
+            and `${HOME}/.esmvaltool` (in that order). If not provided, the
+            file `${HOME}/.esmvaltool/config-user.yml` will be used.
         resume_from: list(str), optional
             Resume one or more previous runs by using preprocessor output files
             from these output directories.
@@ -383,9 +385,15 @@ def run(self,
         """
         from .config import CFG
 
+        # At this point, --config_file is already parsed if a valid file has
+        # been given (see
+        # https://github.com/ESMValGroup/ESMValCore/issues/2280), but no error
+        # has been raised if the file does not exist. Thus, reload the file
+        # here with `load_from_file` to make sure a proper error is raised.
+        CFG.load_from_file(config_file)
+
         recipe = self._get_recipe(recipe)
 
-        CFG.load_from_file(config_file)
         session = CFG.start_session(recipe.stem)
         if check_level is not None:
             session['check_level'] = check_level

esmvalcore/_recipe/check.py

@@ -70,21 +70,21 @@ def diagnostics(diags):
     for name, diagnostic in diags.items():
         if 'scripts' not in diagnostic:
             raise RecipeError(
-                "Missing scripts section in diagnostic {}".format(name))
+                f"Missing scripts section in diagnostic '{name}'.")
         variable_names = tuple(diagnostic.get('variables', {}))
         scripts = diagnostic.get('scripts')
         if scripts is None:
             scripts = {}
         for script_name, script in scripts.items():
             if script_name in variable_names:
                 raise RecipeError(
-                    "Invalid script name {} encountered in diagnostic {}: "
-                    "scripts cannot have the same name as variables.".format(
-                        script_name, name))
+                    f"Invalid script name '{script_name}' encountered "
+                    f"in diagnostic '{name}': scripts cannot have the "
+                    "same name as variables.")
             if not script.get('script'):
                 raise RecipeError(
-                    "No script defined for script {} in diagnostic {}".format(
-                        script_name, name))
+                    f"No script defined for script '{script_name}' in "
+                    f"diagnostic '{name}'.")
 
 
 def duplicate_datasets(
@@ -95,27 +95,31 @@ def duplicate_datasets(
     """Check for duplicate datasets."""
     if not datasets:
         raise RecipeError(
-            "You have not specified any dataset or additional_dataset groups "
-            f"for variable {variable_group} in diagnostic {diagnostic}.")
+            "You have not specified any dataset or additional_dataset "
+            f"groups for variable '{variable_group}' in diagnostic "
+            f"'{diagnostic}'.")
     checked_datasets_ = []
     for dataset in datasets:
         if dataset in checked_datasets_:
             raise RecipeError(
-                f"Duplicate dataset {dataset} for variable {variable_group} "
-                f"in diagnostic {diagnostic}.")
+                f"Duplicate dataset\n{pformat(dataset)}\nfor variable "
+                f"'{variable_group}' in diagnostic '{diagnostic}'.")
         checked_datasets_.append(dataset)
 
 
-def variable(var: dict[str, Any], required_keys: Iterable[str]):
+def variable(
+    var: dict[str, Any],
+    required_keys: Iterable[str],
+    diagnostic: str,
+    variable_group: str
+) -> None:
     """Check variables as derived from recipe."""
     required = set(required_keys)
     missing = required - set(var)
     if missing:
         raise RecipeError(
-            f"Missing keys {missing} in\n"
-            f"{pformat(var)}\n"
-            "for variable {var['variable_group']} in diagnostic "
-            f"{var['diagnostic']}")
+            f"Missing keys {missing} in\n{pformat(var)}\nfor variable "
+            f"'{variable_group}' in diagnostic '{diagnostic}'.")
 
 
 def _log_data_availability_errors(dataset):

esmvalcore/_recipe/recipe_schema.yml

@@ -46,7 +46,7 @@ variable:
 # TODO: add preprocessor item
 
 diagnostic:
-  scripts: any(null(), map(include('script')))
+  scripts: any(null(), map(include('script')), required=False)
   additional_datasets: list(include('dataset'), required=False)
   title: str(required=False)
   description: str(required=False)
@@ -55,4 +55,4 @@ diagnostic:
   variables: map(include('variable'), null(), required=False)
 
 script:
-  script: str()
+  script: str(required=False)

esmvalcore/_recipe/to_datasets.py

@@ -244,6 +244,7 @@ def _get_dataset_facets_from_recipe(
     recipe_variable: dict[str, Any],
     recipe_dataset: dict[str, Any],
     profiles: dict[str, Any],
+    diagnostic_name: str,
     session: Session,
 ) -> tuple[Facets, list[Facets]]:
     """Read the facets for a single dataset definition from the recipe."""
@@ -286,6 +287,8 @@ def _get_dataset_facets_from_recipe(
             'dataset',
             'project',
         ),
+        diagnostic=diagnostic_name,
+        variable_group=variable_group
     )
 
     preprocessor = facets.get('preprocessor', 'default')
@@ -329,6 +332,7 @@ def _get_facets_from_recipe(
             recipe_variable=recipe_variable,
             recipe_dataset=recipe_dataset,
             profiles=profiles,
+            diagnostic_name=diagnostic_name,
             session=session,
         )
 

esmvalcore/cmor/_fixes/cesm/cesm2.py

@@ -78,6 +78,6 @@ def _fix_time(self, cube):
 
         # Fix time coordinate
         time_coord = cube.coord('time')
-        if time_coord.bounds is not None:
-            time_coord.points = time_coord.bounds.mean(axis=-1)
+        if time_coord.has_bounds():
+            time_coord.points = time_coord.core_bounds().mean(axis=-1)
         self.fix_regular_time(cube, coord=time_coord)

esmvalcore/cmor/_fixes/cmip5/hadgem2_cc.py

@@ -1,34 +1,9 @@
-
 """Fix HadGEM2_CC."""
-import numpy as np
-
 from ..fix import Fix
+from .hadgem2_es import AllVars as BaseAllVars
 
 
-class AllVars(Fix):
-    """Fix common errors for all vars."""
-
-    def fix_metadata(self, cubes):
-        """
-        Fix latitude.
-
-        Parameters
-        ----------
-        cube: iris.cube.CubeList
-
-        Returns
-        -------
-        iris.cube.CubeList
-
-        """
-        for cube in cubes:
-            lats = cube.coords('latitude')
-            if lats:
-                lat = cube.coord('latitude')
-                lat.points = np.clip(lat.points, -90., 90.)
-                lat.bounds = np.clip(lat.bounds, -90., 90.)
-
-        return cubes
+AllVars = BaseAllVars
 
 
 class O2(Fix):

esmvalcore/cmor/_fixes/cmip5/hadgem2_es.py

@@ -25,10 +25,10 @@ def fix_metadata(self, cubes):
             lats = cube.coords('latitude')
             if lats:
                 lat = cube.coord('latitude')
-                lat.points = np.clip(lat.points, -90., 90.)
+                lat.points = np.clip(lat.core_points(), -90., 90.)
                 if not lat.has_bounds():
                     lat.guess_bounds()
-                lat.bounds = np.clip(lat.bounds, -90., 90.)
+                lat.bounds = np.clip(lat.core_bounds(), -90., 90.)
 
         return cubes
 

esmvalcore/cmor/_fixes/cmip5/miroc_esm.py

@@ -100,7 +100,7 @@ def fix_metadata(self, cubes):
                                            calendar='gregorian')
                 if cube.coord('time').units != expected_time_units:
                     continue
-                if cube.coord('time').bounds is None:
+                if not cube.coord('time').has_bounds():
                     continue
 
                 # Only apply fix if there is a year < 1 in the first element

esmvalcore/cmor/_fixes/cmip6/access_esm1_5.py

@@ -102,9 +102,9 @@ def fix_metadata(self, cubes):
         """
         cube = self.get_cube_from_list(cubes)
         cube.coord('air_pressure').points = \
-            np.round(cube.coord('air_pressure').points, 0)
+            np.round(cube.coord('air_pressure').core_points(), 0)
         cube.coord('air_pressure').bounds = \
-            np.round(cube.coord('air_pressure').bounds, 0)
+            np.round(cube.coord('air_pressure').core_bounds(), 0)
         return cubes
 
 

esmvalcore/cmor/_fixes/cmip6/cesm2.py

@@ -142,16 +142,12 @@ def fix_metadata(self, cubes):
 
         """
         for cube in cubes:
-            latitude = cube.coord('latitude')
-            if latitude.bounds is None:
-                latitude.guess_bounds()
-            latitude.bounds = latitude.bounds.astype(np.float64)
-            latitude.bounds = np.round(latitude.bounds, 4)
-            longitude = cube.coord('longitude')
-            if longitude.bounds is None:
-                longitude.guess_bounds()
-            longitude.bounds = longitude.bounds.astype(np.float64)
-            longitude.bounds = np.round(longitude.bounds, 4)
+            for coord_name in ['latitude', 'longitude']:
+                coord = cube.coord(coord_name)
+                if not coord.has_bounds():
+                    coord.guess_bounds()
+                coord.bounds = np.round(coord.core_bounds().astype(np.float64),
+                                        4)
 
         return cubes
 

esmvalcore/cmor/_fixes/cmip6/ec_earth3.py

@@ -68,7 +68,7 @@ def fix_metadata(self, cubes):
 
         for cube in cubes:
             latitude = cube.coord('latitude')
-            latitude.points = np.round(latitude.points, 8)
-            latitude.bounds = np.round(latitude.bounds, 8)
+            latitude.points = np.round(latitude.core_points(), 8)
+            latitude.bounds = np.round(latitude.core_bounds(), 8)
 
         return cubes

esmvalcore/cmor/_fixes/cmip6/ec_earth3_veg.py

@@ -72,7 +72,7 @@ def fix_metadata(self, cubes):
 
         for cube in cubes:
             latitude = cube.coord('latitude')
-            latitude.points = np.round(latitude.points, 8)
-            latitude.bounds = np.round(latitude.bounds, 8)
+            latitude.points = np.round(latitude.core_points(), 8)
+            latitude.bounds = np.round(latitude.core_bounds(), 8)
 
         return cubes

esmvalcore/cmor/_fixes/cmip6/gfdl_cm4.py

@@ -62,7 +62,7 @@ def fix_metadata(self, cubes):
         """
         cube = self.get_cube_from_list(cubes)
         try:
-            cube.coord('height')
+            cube.coord('height').attributes.pop('description')
         except iris.exceptions.CoordinateNotFoundError:
             add_scalar_height_coord(cube, 2.0)
         return cubes

esmvalcore/cmor/_fixes/cmip6/giss_e2_1_g.py

@@ -1,6 +1,4 @@
 """Fixes for GISS-E2-1-G model."""
-import dask.array as da
-
 from ..common import ClFixHybridPressureCoord
 from ..fix import Fix
 
@@ -22,7 +20,7 @@ def fix_metadata(self, cubes):
         """
         for cube in cubes:
             if (cube.units == 'degC'
-                    and da.any(cube.core_data().ravel()[:1000] > 100.)):
+                    and cube.core_data().ravel()[:1000].max() > 100.):
                 cube.units = 'K'
                 cube.convert_units(self.vardef.units)
         return cubes

esmvalcore/cmor/_fixes/cmip6/miroc6.py

@@ -28,18 +28,12 @@ def fix_metadata(self, cubes):
         iris.cube.CubeList
         """
         for cube in cubes:
-            latitude = cube.coord('latitude')
-            if latitude.bounds is None:
-                latitude.guess_bounds()
-            latitude.points = latitude.points.astype(np.float32).astype(
-                np.float64)
-            latitude.bounds = latitude.bounds.astype(np.float32).astype(
-                np.float64)
-            longitude = cube.coord('longitude')
-            if longitude.bounds is None:
-                longitude.guess_bounds()
-            longitude.points = longitude.points.astype(np.float32).astype(
-                np.float64)
-            longitude.bounds = longitude.bounds.astype(np.float32).astype(
-                np.float64)
+            for coord_name in ['latitude', 'longitude']:
+                coord = cube.coord(coord_name)
+                coord.points = coord.core_points().astype(np.float32).astype(
+                    np.float64)
+                if not coord.has_bounds():
+                    coord.guess_bounds()
+                coord.bounds = coord.core_bounds().astype(np.float32).astype(
+                    np.float64)
         return cubes

esmvalcore/cmor/_fixes/common.py

@@ -83,7 +83,7 @@ def fix_metadata(self, cubes):
         # This was originally done by iris, but it has to be repeated since
         # a has bounds now
         ap_coord = cube.coord(var_name='ap')
-        if ap_coord.bounds is None:
+        if not ap_coord.has_bounds():
             cube.remove_coord(ap_coord)
             a_coord = cube.coord(var_name='a')
             p0_coord = cube.coord(var_name='p0')

esmvalcore/cmor/_fixes/cordex/cordex_fixes.py

@@ -100,7 +100,7 @@ def fix_metadata(self, cubes):
                 if coord.dtype in ['>f8', '>f4']:
                     coord.points = coord.core_points().astype(
                         np.float64, casting='same_kind')
-                    if coord.bounds is not None:
+                    if coord.has_bounds():
                         coord.bounds = coord.core_bounds().astype(
                             np.float64, casting='same_kind')
         return cubes

esmvalcore/cmor/_fixes/emac/emac.py

@@ -174,7 +174,7 @@ def _fix_alevel(cube, cubes):
         for coord in (ap_coord, b_coord, ps_coord):
             coord.points = coord.core_points().astype(
                 float, casting='same_kind')
-            if coord.bounds is not None:
+            if coord.has_bounds():
                 coord.bounds = coord.core_bounds().astype(
                     float, casting='same_kind')
 

esmvalcore/cmor/_fixes/icon/_base_fixes.py

@@ -450,6 +450,6 @@ def _load_cubes(path):
     @staticmethod
     def _set_range_in_0_360(lon_coord):
         """Convert longitude coordinate to [0, 360]."""
-        lon_coord.points = (lon_coord.points + 360.0) % 360.0
-        if lon_coord.bounds is not None:
-            lon_coord.bounds = (lon_coord.bounds + 360.0) % 360.0
+        lon_coord.points = (lon_coord.core_points() + 360.0) % 360.0
+        if lon_coord.has_bounds():
+            lon_coord.bounds = (lon_coord.core_bounds() + 360.0) % 360.0