feat: add published-path parameter to the standardise_validate script

linz · Nov 3, 2024 · 52f89ba · 52f89ba
1 parent fc371bb
commit 52f89ba
Show file tree

Hide file tree

Showing 6 changed files with 122 additions and 9 deletions.
diff --git a/scripts/stac/imagery/create_stac.py b/scripts/stac/imagery/create_stac.py
@@ -1,5 +1,5 @@
 import json
-import os
+from os import environ, path
 from typing import Any
 
 from linz_logger import get_log
@@ -108,9 +108,6 @@ def create_item(
     """
     item = create_base_item(asset_path, gdal_version, current_datetime, published_path)
 
-    if not gdalinfo_result:
-        gdalinfo_result = gdal_info(asset_path)
-
     if derived_from is not None:
         for derived in derived_from:
             derived_item_content = read(derived)
@@ -129,7 +126,7 @@ def create_item(
             )
 
     item.update_datetime(start_datetime, end_datetime)
-    item.update_spatial(*get_extents(gdalinfo_result))
+    item.update_spatial(*get_extents(gdalinfo_result or gdal_info(asset_path)))
     item.add_collection(collection_id)
 
     get_log().info("ImageryItem created", path=asset_path)
@@ -148,7 +145,7 @@ def create_base_item(asset_path: str, gdal_version: str, current_datetime: str,
         An ImageryItem with basic information.
     """
 
-    if (topo_imagery_hash := os.environ.get("GIT_HASH")) is not None:
+    if (topo_imagery_hash := environ.get("GIT_HASH")) is not None:
         commit_url = f"https://github.com/linz/topo-imagery/commit/{topo_imagery_hash}"
     else:
         commit_url = "GIT_HASH not specified"
@@ -157,7 +154,7 @@ def create_base_item(asset_path: str, gdal_version: str, current_datetime: str,
         **{
             "processing:datetime": current_datetime,
             "processing:software": STACProcessingSoftware(**{"gdal": gdal_version, "linz/topo-imagery": commit_url}),
-            "processing:version": os.environ.get("GIT_VERSION", "GIT_VERSION not specified"),
+            "processing:version": environ.get("GIT_VERSION", "GIT_VERSION not specified"),
         }
     )
 
@@ -168,7 +165,7 @@ def create_base_item(asset_path: str, gdal_version: str, current_datetime: str,
     if published_path:
         # FIXME: make this try/catch nicer
         try:
-            existing_item_content = read(os.path.join(published_path, f"{id_}.json"))
+            existing_item_content = read(path.join(published_path, f"{id_}.json"))
             existing_item = json.loads(existing_item_content.decode("UTF-8"))
             created_datetime = existing_item["properties"]["created"]
             try:
@@ -193,7 +190,7 @@ def create_base_item(asset_path: str, gdal_version: str, current_datetime: str,
 
     stac_asset = STACAsset(
         **{
-            "href": os.path.join(".", os.path.basename(asset_path)),
+            "href": path.join(".", path.basename(asset_path)),
             "file:checksum": multihash_as_hex(file_content),
             "created": created_datetime,
             "updated": updated_datetime,

diff --git a/scripts/stac/imagery/tests/create_stac_test.py b/scripts/stac/imagery/tests/create_stac_test.py
@@ -11,6 +11,34 @@
 from scripts.tests.datetimes_test import any_epoch_datetime_string
 
 
+def test_create_item(subtests: SubTests) -> None:
+    fake_gdal_info: GdalInfo = cast(
+        GdalInfo, {"wgs84Extent": {"type": "Polygon", "coordinates": [[[0, 1], [1, 1], [1, 0], [0, 0]]]}}
+    )
+    current_datetime = any_epoch_datetime_string()
+    item = create_item(
+        "./scripts/tests/data/empty.tiff",
+        "",
+        "",
+        "abc123",
+        "any GDAL version",
+        current_datetime,
+        fake_gdal_info,
+    )
+
+    with subtests.test(msg="properties.created"):
+        assert item.stac["properties"]["created"] == current_datetime
+
+    with subtests.test(msg="properties.updated"):
+        assert item.stac["properties"]["updated"] == current_datetime
+
+    with subtests.test(msg="assets.visual.created"):
+        assert item.stac["assets"]["visual"]["created"] == current_datetime
+
+    with subtests.test(msg="assets.visual.updated"):
+        assert item.stac["assets"]["visual"]["updated"] == current_datetime
+
+
 def test_create_item_with_derived_from(tmp_path: Path) -> None:
     derived_from_path = tmp_path / "derived_from_item.json"
     fake_item = {

diff --git a/scripts/standardise_validate.py b/scripts/standardise_validate.py
@@ -29,6 +29,12 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument(
         "--from-file", dest="from_file", required=True, help="The path to a json file containing the input tiffs"
     )
+    parser.add_argument(
+        "--published-path",
+        dest="published_path",
+        help=("The path of the published dataset. Example: 's3://nz-imagery/wellington/porirua_2024_0.1m/rgb/2193/'"),
+        required=False,
+    )
     parser.add_argument("--source-epsg", dest="source_epsg", required=True, help="The EPSG code of the source imagery")
     parser.add_argument(
         "--target-epsg",
@@ -159,6 +165,7 @@ def main() -> None:
                 arguments.current_datetime,
                 file.get_gdalinfo(),
                 file.get_derived_from_paths(),
+                arguments.published_path,
             )
             write(stac_item_path, dict_to_json_bytes(item.stac), content_type=ContentType.GEOJSON.value)
             get_log().info("stac_saved", path=stac_item_path)

diff --git a/scripts/tests/data/BQ25_10000_0305.json b/scripts/tests/data/BQ25_10000_0305.json
@@ -0,0 +1,17 @@
+{
+  "type": "Feature",
+  "id": "BQ25_10000_0305",
+  "assets": {
+    "visual": {
+      "href": "./BQ25_10000_0305.tiff",
+      "type": "image/tiff; application=geotiff; profile=cloud-optimized",
+      "file:checksum": "12205f300ac3bd1d289da1517144d4851050e544c43c58c23ccfcc1f6968f764a45a",
+      "created": "2024-10-01T10:31:00Z",
+      "updated": "2024-10-01T10:31:00Z"
+    }
+  },
+  "properties": {
+    "created": "2024-10-01T10:31:00Z",
+    "updated": "2024-10-01T10:31:00Z"
+  }
+}
diff --git a/scripts/tests/data/output/BQ25_10000_0305.json b/scripts/tests/data/output/BQ25_10000_0305.json
@@ -0,0 +1,58 @@
+{
+  "type": "Feature",
+  "stac_version": "1.0.0",
+  "id": "BQ25_10000_0305",
+  "links": [
+    {
+      "href": "./BQ25_10000_0305.json",
+      "rel": "self",
+      "type": "application/geo+json"
+    },
+    {
+      "href": "./collection.json",
+      "rel": "collection",
+      "type": "application/json"
+    },
+    { "href": "./collection.json", "rel": "parent", "type": "application/json" }
+  ],
+  "assets": {
+    "visual": {
+      "href": "/tmp/BQ25_10000_0305.tiff",
+      "file:checksum": "1220a2fb26080135bb39515795a61046db5155ffecf74901a1ac544a983839ae6b86",
+      "created": "2024-10-01T10:31:00Z",
+      "updated": "2024-10-28T21:42:00Z",
+      "type": "image/tiff; application=geotiff; profile=cloud-optimized"
+    }
+  },
+  "stac_extensions": [
+    "https://stac-extensions.github.io/file/v2.0.0/schema.json",
+    "https://stac-extensions.github.io/processing/v1.2.0/schema.json"
+  ],
+  "properties": {
+    "created": "2024-10-01T10:31:00Z",
+    "updated": "2025-01-03T08:24:00Z",
+    "processing:datetime": "2025-01-03T08:24:00Z",
+    "processing:software": {
+      "gdal": "GDAL 3.9.0, released 2024/05/07",
+      "linz/topo-imagery": "https://github.com/linz/topo-imagery/commit/"
+    },
+    "processing:version": "",
+    "start_datetime": "2022-12-31T11:00:00Z",
+    "end_datetime": "2022-12-31T11:00:00Z",
+    "datetime": null
+  },
+  "geometry": {
+    "type": "Polygon",
+    "coordinates": [
+      [
+        [173.0860059, -41.3030235],
+        [173.0860913, -41.367879],
+        [173.1434853, -41.3678217],
+        [173.143343, -41.3029664],
+        [173.0860059, -41.3030235]
+      ]
+    ]
+  },
+  "bbox": [173.0860059, -41.367879, 173.1434853, -41.3029664],
+  "collection": "123"
+}
diff --git a/scripts/tests/data/resupply.json b/scripts/tests/data/resupply.json
@@ -0,0 +1,6 @@
+[
+  {
+    "output": "BQ25_10000_0305",
+    "input": ["./tests/data/BQ25_10000_0305.tiff"]
+  }
+]