diff --git a/cube_builder/utils/index_generator.py b/cube_builder/utils/index_generator.py index 8fef5e7..3c3309d 100644 --- a/cube_builder/utils/index_generator.py +++ b/cube_builder/utils/index_generator.py @@ -8,6 +8,7 @@ """Simple data cube band generator.""" +import logging from typing import List, Dict import numpy @@ -35,7 +36,7 @@ def generate_band_indexes(cube: Collection, scenes: dict, period: str, tile_id: Returns: A dict values with generated bands. """ - from .utils import SmartDataSet, build_cube_path + from .processing import SmartDataSet, build_cube_path, generate_cogs cube_band_indexes: List[Band] = [] @@ -57,6 +58,9 @@ def generate_band_indexes(cube: Collection, scenes: dict, period: str, tile_id: profile = map_data_set_context[band_name].dataset.profile.copy() blocks = list(map_data_set_context[band_name].dataset.block_windows()) + if not blocks or profile is None: + raise RuntimeError('Can\t generate band indexes since profile/blocks is None.') + output = dict() for band_index in cube_band_indexes: @@ -76,7 +80,7 @@ def generate_band_indexes(cube: Collection, scenes: dict, period: str, tile_id: custom_band_path = build_cube_path(cube.name, period, tile_id, version=cube.version, band=band_name) output_dataset = SmartDataSet(str(custom_band_path), mode='w', **profile) - logging.info(f'Generating band {band_name} for cube {cube.name}...') + logging.info(f'Generating band {band_name} for cube {cube.name} -{custom_band_path.stem}...') for _, window in blocks: machine_context = { @@ -89,13 +93,18 @@ def generate_band_indexes(cube: Collection, scenes: dict, period: str, tile_id: result = execute(expr, context=machine_context) raster = result[band_name] raster[raster == numpy.ma.masked] = profile['nodata'] - # Persist the expected band data type to cast value safelly. + + # Persist the expected band data type to cast value safely. + # TODO: Should we use consider band min_value/max_value? raster[raster < data_type_min_value] = data_type_min_value raster[raster > data_type_max_value] = data_type_max_value output_dataset.dataset.write(raster.astype(band_data_type), window=window, indexes=1) output_dataset.close() + + generate_cogs(str(custom_band_path), str(custom_band_path)) + output[band_name] = str(custom_band_path) - return output \ No newline at end of file + return output diff --git a/cube_builder/utils/processing.py b/cube_builder/utils/processing.py index bfa730d..4abe744 100644 --- a/cube_builder/utils/processing.py +++ b/cube_builder/utils/processing.py @@ -31,11 +31,12 @@ from numpngw import write_png from rasterio import Affine, MemoryFile from rasterio.warp import Resampling, reproject - -# Builder from rio_cogeo.cogeo import cog_translate from rio_cogeo.profiles import cog_profiles +# Builder +from .index_generator import generate_band_indexes + from ..config import Config @@ -497,16 +498,13 @@ def __exit__(self, exc_type, exc_val, exc_tb): def close(self): """Close rasterio data set.""" if not self.dataset.closed: - logging.warning('Closing dataset {}'.format(str(self.path))) + logging.debug('Closing dataset {}'.format(str(self.path))) if self.mode == 'w' and self.tags: self.dataset.update_tags(**self.tags) self.dataset.close() - if self.mode == 'w': - generate_cogs(str(path), str(path)) - def compute_data_set_stats(file_path: str) -> Tuple[float, float]: """Compute data set efficacy and cloud ratio. @@ -938,7 +936,7 @@ def publish_datacube(cube, bands, tile_id, period, scenes, cloudratio, band_map, item, _ = get_or_create_model(Item, defaults=item_data, name=item_id, collection_id=cube.id) item.cloud_cover = cloudratio - assets = item.assets or dict() + assets = deepcopy(item.assets) or dict() assets.update( thumbnail=create_asset_definition( href=quick_look_file.replace(Config.DATA_DIR, ''), @@ -1024,7 +1022,7 @@ def publish_merge(bands, datacube, tile_id, date, scenes, band_map): extent = to_shape(item.geom) if item.geom else None min_convex_hull = to_shape(item.min_convex_hull) if item.min_convex_hull else None - assets = item.assets or dict() + assets = deepcopy(item.assets) or dict() assets.update( thumbnail=create_asset_definition( @@ -1239,7 +1237,7 @@ def create_asset_definition(href: str, mime_type: str, role: List[str], absolute chunk_x, chunk_y = data_set.profile.get('blockxsize'), data_set.profile.get('blockxsize') if chunk_x is None or chunk_x is None: - raise RuntimeError('Can\'t compute raster chunk size. Is it a tiled/ valid Cloud Optimized GeoTIFF?') + return asset asset['bdc:chunk_size'] = dict(x=chunk_x, y=chunk_y)