Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Creo missing products #584

Closed
wants to merge 5 commits into from
Closed

Creo missing products #584

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6bca9d2
Log in if-case where that gives misterious missing products. https://…
EmileSonneveld Nov 7, 2023
5cb7d0c
query_offline should probably list products that are not AVAILABLE an…
EmileSonneveld Nov 7, 2023
08fc9c6
Only warn when STATUS=OFFLINE products do not have an ONLINE equivale…
EmileSonneveld Nov 13, 2023
6c58f43
Minimize diff.
EmileSonneveld Nov 13, 2023
b9183d5
Fix test.
EmileSonneveld Nov 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 25 additions & 4 deletions openeogeotrellis/layercatalog.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -971,7 +971,7 @@ def _merge_layers_with_common_name(metadata: CatalogDict):
return metadata


def query_jvm_opensearch_client(open_search_client, collection_id, _query_kwargs, processing_level=""):
def query_jvm_opensearch_client(open_search_client, collection_id, _query_kwargs, processing_level="", attribute_values_dict=None):
jvm = get_jvm()
fromDate = jvm.java.time.ZonedDateTime.parse(str(_query_kwargs["start_date"]).replace(" ", "T") + "Z")
toDate = jvm.java.time.ZonedDateTime.parse(str(_query_kwargs["end_date"]).replace(" ", "T") + "Z")
Expand All @@ -980,7 +980,8 @@ def query_jvm_opensearch_client(open_search_client, collection_id, _query_kwargs
float(_query_kwargs["brx"]), float(_query_kwargs["uly"]))
crs = jvm.geotrellis.proj4.CRS.fromEpsgCode(4326)
bbox = jvm.geotrellis.vector.ProjectedExtent(extent, crs)
attribute_values_dict = {}

attribute_values_dict = deepcopy(attribute_values_dict or {})
if "cldPrcnt" in _query_kwargs:
attribute_values_dict["eo:cloud_cover"] = _query_kwargs["cldPrcnt"]
attributeV_values = jvm.PythonUtils.toScalaMap(attribute_values_dict)
Expand Down Expand Up @@ -1040,8 +1041,28 @@ def check_missing_products(

method = check_data.get("method")
if method == "creo":
creo_catalog = CreoCatalogClient(**check_data["creo_catalog"])
missing = [p.getProductId() for p in creo_catalog.query_offline(**query_kwargs)]
jvm = get_jvm()
open_search_client = jvm.org.openeo.opensearch.backends.CreodiasClient.apply()
processing_level = ""
if "level" in check_data["creo_catalog"]:
processing_level = check_data["creo_catalog"]["level"]
offline_tiles = query_jvm_opensearch_client(
open_search_client=open_search_client,
collection_id=check_data["creo_catalog"]["mission"],
_query_kwargs=query_kwargs,
processing_level=processing_level,

# Offline products (in the long-term preservation archive or in the remote repository)
attribute_values_dict={"status": "OFFLINE"}
)
online_tiles = query_jvm_opensearch_client(
open_search_client=open_search_client,
collection_id=check_data["creo_catalog"]["mission"],
_query_kwargs=query_kwargs,
processing_level=processing_level,
attribute_values_dict={"status": "ONLINE"}
)
missing = list(offline_tiles.difference(online_tiles))
elif method == "terrascope":
jvm = get_jvm()

Expand Down
41 changes: 24 additions & 17 deletions tests/test_api_result.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
import os
import shutil
import textwrap
import urllib.request
import urllib.parse
import urllib.request
from pathlib import Path
from typing import List, Union, Sequence

Expand Down Expand Up @@ -42,15 +42,15 @@
)
from openeogeotrellis.backend import JOB_METADATA_FILENAME
from openeogeotrellis.job_registry import ZkJobRegistry
from openeogeotrellis.layercatalog import get_layer_catalog
from openeogeotrellis.testing import random_name, KazooClientMock, config_overrides
from openeogeotrellis.utils import (
UtcNowClock,
drop_empty_from_aggregate_polygon_result,
get_jvm,
is_package_available,
)
from openeogeotrellis.vault import OAuthCredentials
from .data import get_test_data_file
from tests.data import get_test_data_file
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure this is an improvement over the original from .data import get_test_data_file (which is also not ideal by the way). This import makes it look like there is some top-level tests package in the venv while the relative import makes it more clear it is just from a sibling file.

Is there a particular reason you changed this?


_log = logging.getLogger(__name__)

Expand All @@ -75,6 +75,13 @@ def set_all(properties: dict):
set_all(orig_properties)


@pytest.fixture
def catalog(vault):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why is this fixture added? It doesn't seem to be used

catalog = get_layer_catalog(vault)
catalog.set_default_sentinel_hub_credentials(client_id="???", client_secret="!!!")
return catalog


def test_execute_math_basic(api100):
res = api100.check_result({"add": {"process_id": "add", "arguments": {"x": 3, "y": 5}, "result": True}})
assert res.json == 8
Expand Down Expand Up @@ -1903,7 +1910,7 @@ def feature_collection(cls, features: List[dict]) -> dict:
}


def test_extra_validation_creo(api100, requests_mock):
def test_extra_validation_creo(api100):
pg = {"lc": {
"process_id": "load_collection",
"arguments": {
Expand All @@ -1915,20 +1922,20 @@ def test_extra_validation_creo(api100, requests_mock):
"result": True
}}

requests_mock.get(
"https://catalogue.dataspace.copernicus.eu/resto/api/collections/Sentinel2/search.json?productType=L2A&startDate=2020-03-01T00%3A00%3A00&cloudCover=%5B0%2C50%5D&page=1&maxRecords=100&sortParam=startDate&sortOrder=ascending&status=all&dataset=ESA-DATASET&completionDate=2020-03-10T23%3A59%3A59.999999&geometry=POLYGON+%28%28-87+68%2C+-86+68%2C+-86+67%2C+-87+67%2C+-87+68%29%29",
json=CreoApiMocker.feature_collection(features=[{"tile_id": "16WEV"}, {"tile_id": "16WDA", "status": 31}]),
)
requests_mock.get(
"https://catalogue.dataspace.copernicus.eu/resto/api/collections/Sentinel2/search.json?productType=L2A&startDate=2020-03-01T00%3A00%3A00&cloudCover=%5B0%2C50%5D&page=2&maxRecords=100&sortParam=startDate&sortOrder=ascending&status=all&dataset=ESA-DATASET&completionDate=2020-03-10T23%3A59%3A59.999999&geometry=POLYGON+%28%28-87+68%2C+-86+68%2C+-86+67%2C+-87+67%2C+-87+68%29%29",
json=CreoApiMocker.feature_collection(features=[]),
)
# noinspection PyUnusedLocal
def mock_query_jvm_opensearch_client(open_search_client, collection_id, _query_kwargs,
processing_level="", attribute_values_dict=None):
if attribute_values_dict["status"] == "OFFLINE":
return {"16WEA", "17WEA"}
elif attribute_values_dict["status"] == "ONLINE":
return {"17WEA", "18WEA"}
else:
raise Exception("Unknown open_search_client: " + str(open_search_client))

response = api100.validation(pg)
assert response.json == {'errors': [
{'code': 'MissingProduct',
'message': "Tile 'S2A_MSIL2A_20200301T173231_N0209_R055_T16WDA_20200301T210331' in collection 'SENTINEL2_L2A_CREO' is not available."}
]}
with mock.patch("openeogeotrellis.layercatalog.query_jvm_opensearch_client", new=mock_query_jvm_opensearch_client):
response = api100.validation(pg)
assert response.json == {'errors': [
{'code': 'MissingProduct', 'message': "Tile '16WEA' in collection 'SENTINEL2_L2A_CREO' is not available."}]}


@pytest.fixture
Expand Down