GribInternalError: ('Passed array is too small (-6).', -6)

[tommylees112]

I am trying to open a .grib file that we are using for writing tests for our repository with the ECMWF Summer of Weather Code. I wondered if we were missing anything that would allow us to open the written file?

from pathlib import Path
import xarray as xr
import numpy as np
import pandas as pd
import cfgrib


def make_dummy_seas5_data(date_str: str) -> xr.Dataset:
    initialisation_date = pd.date_range(start=date_str, periods=1, freq='M')
    number = [i for i in range(0, 26)]  # corresponds to ensemble number (51)
    lat = np.linspace(-5.175003, -5.202, 36)
    lon = np.linspace(33.5, 42.25, 45)
    forecast_horizon = np.array(
        [2419200000000000, 2592000000000000, 2678400000000000,
         5097600000000000, 5270400000000000, 5356800000000000,
         7689600000000000, 7776000000000000, 7862400000000000,
         7948800000000000, 10368000000000000, 10454400000000000,
         10540800000000000, 10627200000000000, 12960000000000000,
         13046400000000000, 13219200000000000, 15638400000000000,
         15724800000000000, 15811200000000000, 15897600000000000,
         18316800000000000, 18489600000000000, 18576000000000000],
        dtype='timedelta64[ns]'
    )
    valid_time = initialisation_date[:, np.newaxis] + forecast_horizon
    precip = np.ones(
        shape=(
            len(number), len(initialisation_date),
            len(forecast_horizon), len(lat), len(lon)
        )
    )

    ds = xr.Dataset(
        {'precip': (
            ['number', 'initialisation_date', 'forecast_horizon', 'lat', 'lon'],
            precip
        )},
        coords={
            'lon': lon,
            'lat': lat,
            'initialisation_date': initialisation_date,
            'number': number,
            'forecast_horizon': forecast_horizon,
            'valid_time': (['initialisation_date', 'step'], valid_time)
        }
    )
    return ds


def save_dummy_seas5(tmp_path,
                     date_str,
                     to_grib=False,
                     dataset='seasonal-monthly-pressure-levels',
                     variable='temperature') -> Path:
    """
    filename structure:
     data/raw/seasonal-monthly-pressure-levels/temperature/2017/01.grib
    """
    year = pd.to_datetime(date_str).year
    month = pd.to_datetime(date_str).month
    out_dir = tmp_path / 'data' / 'raw' / 's5' / dataset / variable / str(year)
    if not out_dir.exists():
        out_dir.mkdir(exist_ok=True, parents=True)

    ds = make_dummy_seas5_data(date_str)
    if to_grib:
        cfgrib.to_grib(
            ds, out_dir / f'{month:02}.grib',
            grib_keys={'edition': 2, 'gridType': 'regular_ll'}
        )
    else:
        ds.to_netcdf(out_dir / f'{month:02}.nc')

    return out_dir


data_path = save_dummy_seas5(tmp_path,
                             '2018-01-01',
                             to_grib=True,
                             dataset='seasonal-monthly-pressure-levels',
                             variable='temperature')

xr.open_dataset(out_dir / '01.grib', engine='cfgrib')

The error message

In [73]: xr.open_dataset(p2 / '01.grib', engine='cfgrib')
---------------------------------------------------------------------------
GribInternalError                         Traceback (most recent call last)
<ipython-input-73-dcbf6e8f294e> in <module>
----> 1 xr.open_dataset(p2 / '01.grib', engine='cfgrib')

~/miniconda3/envs/crop/lib/python3.7/site-packages/xarray/backends/api.py in open_dataset(filename_or_obj, group, decode_cf, mask_and_scale, decode_times, autoclose, concat_characters, decode_coords, engine, chunks, lock, cache, drop_variables, backend_kwargs, use_cftime)
    435         elif engine == 'cfgrib':
    436             store = backends.CfGribDataStore(
--> 437                 filename_or_obj, lock=lock, **backend_kwargs)
    438
    439     else:

~/miniconda3/envs/crop/lib/python3.7/site-packages/xarray/backends/cfgrib_.py in __init__(self, filename, lock, **backend_kwargs)
     38             lock = ECCODES_LOCK
     39         self.lock = ensure_lock(lock)
---> 40         self.ds = cfgrib.open_file(filename, **backend_kwargs)
     41
     42     def open_store_variable(self, name, var):

~/miniconda3/envs/crop/lib/python3.7/site-packages/cfgrib/dataset.py in open_file(path, grib_errors, indexpath, filter_by_keys, **kwargs)
    602     """Open a GRIB file as a ``cfgrib.Dataset``."""
    603     index = open_fileindex(path, grib_errors, indexpath, filter_by_keys)
--> 604     return Dataset(*build_dataset_components(index, **kwargs))

~/miniconda3/envs/crop/lib/python3.7/site-packages/cfgrib/dataset.py in build_dataset_components(index, errors, encode_cf, timestamp, squeeze, log)
    528         try:
    529             dims, data_var, coord_vars = build_variable_components(
--> 530                 var_index, encode_cf, filter_by_keys, errors=errors, squeeze=squeeze
    531             )
    532         except DatasetBuildError as ex:

~/miniconda3/envs/crop/lib/python3.7/site-packages/cfgrib/dataset.py in build_variable_components(index, encode_cf, filter_by_keys, log, errors, squeeze)
    463     header_shape = tuple(coord_vars[d].data.size for d in header_dimensions)
    464
--> 465     geo_dims, geo_shape, geo_coord_vars = build_geography_coordinates(index, encode_cf, errors)
    466     dimensions = header_dimensions + geo_dims
    467     shape = header_shape + geo_shape

~/miniconda3/envs/crop/lib/python3.7/site-packages/cfgrib/dataset.py in build_geography_coordinates(index, encode_cf, errors, log)
    355         geo_dims = ('latitude', 'longitude')  # type: T.Tuple[str, ...]
    356         geo_shape = (index.getone('Ny'), index.getone('Nx'))  # type: T.Tuple[int, ...]
--> 357         latitudes = np.array(first['distinctLatitudes'])
    358         geo_coord_vars['latitude'] = Variable(
    359             dimensions=('latitude',), data=latitudes, attributes=COORD_ATTRS['latitude'].copy()

~/miniconda3/envs/crop/lib/python3.7/site-packages/cfgrib/messages.py in __getitem__(self, item)
    174             return getter(self)
    175         else:
--> 176             return super(ComputedKeysMessage, self).__getitem__(item)
    177
    178     def __iter__(self):

~/miniconda3/envs/crop/lib/python3.7/site-packages/cfgrib/messages.py in __getitem__(self, item)
    126     def __getitem__(self, item):
    127         # type: (str) -> T.Any
--> 128         return self.message_get(item)
    129
    130     def __setitem__(self, item, value):

~/miniconda3/envs/crop/lib/python3.7/site-packages/cfgrib/messages.py in message_get(self, item, key_type, default)
     90         key = item
     91         try:
---> 92             values = eccodes.codes_get_array(self.codes_id, key, key_type)
     93             if values is None:
     94                 values = ['unsupported_key_type']

~/miniconda3/envs/crop/lib/python3.7/site-packages/cfgrib/bindings.py in codes_get_array(handle, key, key_type, size, length, log)
    365         return codes_get_long_array(handle, key, size)
    366     elif key_type == float:
--> 367         return codes_get_double_array(handle, key, size)
    368     elif key_type == str:
    369         return codes_get_string_array(handle, key, size, length=length)

~/miniconda3/envs/crop/lib/python3.7/site-packages/cfgrib/bindings.py in codes_get_double_array(handle, key, size)
    283     values = ffi.new('double[]', size)
    284     size_p = ffi.new('size_t *', size)
--> 285     _codes_get_double_array(handle, key.encode(ENC), values, size_p)
    286     return list(values)
    287

~/miniconda3/envs/crop/lib/python3.7/site-packages/cfgrib/bindings.py in wrapper(*args)
    163                 raise ERROR_MAP[code](code)
    164             else:
--> 165                 raise GribInternalError(code)
    166
    167     return wrapper

GribInternalError: ('Passed array is too small (-6).', -6)

[alexamici]

@tommylees112 the same considerations I expressed in comment #91 (comment) for netCDF apply here for an arbitrary xarray.Dataset:

The problem it that in order to save arbitrary netCDF files to GRIB with cfgrib you must specify all the grib_keys that are not automatically computed. At the moment very few keys are computed and you need to set almost all the values as you would do with ecCodes.

Saving a to grib a GRIB file opened with cfgrib works more often because the dataset keeps the GRIB keys of the original file, but if you start from a netCDF you need to do the work manually and you need to know the GRIB data format.

[tommylees112]

so the best bet is to open another grib file, copy all the xr.Dataset.attrs and then write with those attrs saved to the new object? it's only for our test suite! the pipeline itself writes everything to netcdf but i want to check it works with grib without having to download a grib file from the CDS API

[alexamici]

Sorry I didn't notice your comment, I guess you already figured out that your approach is the right one.

I'll close the issue as duplicate of #18