Close #77 - Generate band indexes relying on BDC-Catalog Bands metada…

…ta expression

cube_builder/utils/index_generator.py

@@ -8,6 +8,7 @@
 
 """Simple data cube band generator."""
 
+import logging
 from typing import List, Dict
 
 import numpy
@@ -35,7 +36,7 @@ def generate_band_indexes(cube: Collection, scenes: dict, period: str, tile_id:
     Returns:
         A dict values with generated bands.
     """
-    from .utils import SmartDataSet, build_cube_path
+    from .processing import SmartDataSet, build_cube_path, generate_cogs
 
     cube_band_indexes: List[Band] = []
 
@@ -57,6 +58,9 @@ def generate_band_indexes(cube: Collection, scenes: dict, period: str, tile_id:
             profile = map_data_set_context[band_name].dataset.profile.copy()
             blocks = list(map_data_set_context[band_name].dataset.block_windows())
 
+    if not blocks or profile is None:
+        raise RuntimeError('Can\t generate band indexes since profile/blocks is None.')
+
     output = dict()
 
     for band_index in cube_band_indexes:
@@ -76,7 +80,7 @@ def generate_band_indexes(cube: Collection, scenes: dict, period: str, tile_id:
         custom_band_path = build_cube_path(cube.name, period, tile_id, version=cube.version, band=band_name)
 
         output_dataset = SmartDataSet(str(custom_band_path), mode='w', **profile)
-        logging.info(f'Generating band {band_name} for cube {cube.name}...')
+        logging.info(f'Generating band {band_name} for cube {cube.name} -{custom_band_path.stem}...')
 
         for _, window in blocks:
             machine_context = {
@@ -89,13 +93,18 @@ def generate_band_indexes(cube: Collection, scenes: dict, period: str, tile_id:
             result = execute(expr, context=machine_context)
             raster = result[band_name]
             raster[raster == numpy.ma.masked] = profile['nodata']
-            # Persist the expected band data type to cast value safelly.
+
+            # Persist the expected band data type to cast value safely.
+            # TODO: Should we use consider band min_value/max_value?
             raster[raster < data_type_min_value] = data_type_min_value
             raster[raster > data_type_max_value] = data_type_max_value
 
             output_dataset.dataset.write(raster.astype(band_data_type), window=window, indexes=1)
 
         output_dataset.close()
+
+        generate_cogs(str(custom_band_path), str(custom_band_path))
+
         output[band_name] = str(custom_band_path)
 
-    return output
+    return output

cube_builder/utils/processing.py

@@ -31,11 +31,12 @@
 from numpngw import write_png
 from rasterio import Affine, MemoryFile
 from rasterio.warp import Resampling, reproject
-
-# Builder
 from rio_cogeo.cogeo import cog_translate
 from rio_cogeo.profiles import cog_profiles
 
+# Builder
+from .index_generator import generate_band_indexes
+
 from ..config import Config
 
 
@@ -497,16 +498,13 @@ def __exit__(self, exc_type, exc_val, exc_tb):
     def close(self):
         """Close rasterio data set."""
         if not self.dataset.closed:
-            logging.warning('Closing dataset {}'.format(str(self.path)))
+            logging.debug('Closing dataset {}'.format(str(self.path)))
 
             if self.mode == 'w' and self.tags:
                 self.dataset.update_tags(**self.tags)
 
             self.dataset.close()
 
-            if self.mode == 'w':
-                generate_cogs(str(path), str(path))
-
 
 def compute_data_set_stats(file_path: str) -> Tuple[float, float]:
     """Compute data set efficacy and cloud ratio.
@@ -938,7 +936,7 @@ def publish_datacube(cube, bands, tile_id, period, scenes, cloudratio, band_map,
             item, _ = get_or_create_model(Item, defaults=item_data, name=item_id, collection_id=cube.id)
             item.cloud_cover = cloudratio
 
-            assets = item.assets or dict()
+            assets = deepcopy(item.assets) or dict()
             assets.update(
                 thumbnail=create_asset_definition(
                     href=quick_look_file.replace(Config.DATA_DIR, ''),
@@ -1024,7 +1022,7 @@ def publish_merge(bands, datacube, tile_id, date, scenes, band_map):
         extent = to_shape(item.geom) if item.geom else None
         min_convex_hull = to_shape(item.min_convex_hull) if item.min_convex_hull else None
 
-        assets = item.assets or dict()
+        assets = deepcopy(item.assets) or dict()
 
         assets.update(
             thumbnail=create_asset_definition(
@@ -1239,7 +1237,7 @@ def create_asset_definition(href: str, mime_type: str, role: List[str], absolute
             chunk_x, chunk_y = data_set.profile.get('blockxsize'), data_set.profile.get('blockxsize')
 
             if chunk_x is None or chunk_x is None:
-                raise RuntimeError('Can\'t compute raster chunk size. Is it a tiled/ valid Cloud Optimized GeoTIFF?')
+                return asset
 
             asset['bdc:chunk_size'] = dict(x=chunk_x, y=chunk_y)