25 support tileid filter with wildcard on cdse

src/main/scala/org/openeo/opensearch/OpenSearchResponses.scala

@@ -641,11 +641,11 @@ object OpenSearchResponses {
       geometry
     }
 
-    def parse(json: String, dedup: Boolean = false): FeatureCollection = {
+    def parse(json: String, dedup: Boolean = false, tileIdPattern: Option[String] = None): FeatureCollection = {
       implicit val decodeFeature: Decoder[Feature] = new Decoder[Feature] {
         override def apply(c: HCursor): Decoder.Result[Feature] = {
           for {
-            id <- c.downField("properties").downField("productIdentifier").as[String]
+            id <- c.downField("properties").downField("productIdentifier").as[String] // TODO: that's not the feature ID
             geometry <- c.downField("geometry").as[Json]
             nominalDate <- c.downField("properties").downField("startDate").as[ZonedDateTime]
             links <- c.downField("properties").downField("links").as[Array[Link]]
@@ -675,15 +675,15 @@ object OpenSearchResponses {
         }
       }
 
-
-
       implicit val decodeFeatureCollection: Decoder[FeatureCollection] = new Decoder[FeatureCollection] {
         override def apply(c: HCursor): Decoder.Result[FeatureCollection] = {
           for {
             itemsPerPage <- c.downField("properties").downField("itemsPerPage").as[Int]
             features <- c.downField("features").as[Array[Feature]]
           } yield {
-            val featuresFiltered = if (dedup) dedupFeatures(removePhoebusFeatures(features)) else features
+            val featuresFiltered =
+              if (dedup) dedupFeatures(removePhoebusFeatures(retainTileIdPattern(features, tileIdPattern)))
+              else retainTileIdPattern(features, tileIdPattern)
             FeatureCollection(itemsPerPage, featuresFiltered)
           }
         }
@@ -692,6 +692,20 @@ object OpenSearchResponses {
       decode[FeatureCollection](json)
         .valueOr(e => throw new IllegalArgumentException(s"${e.show} while parsing '$json'", e))
     }
+
+    private def retainTileIdPattern(features: Array[Feature], tileIdPattern: Option[String]): Array[Feature] =
+      tileIdPattern match {
+        case Some(pattern) => features.filter(feature => feature.tileID match {
+          case Some(tileId) =>
+            val matchesPattern = tileId matches pattern.replace("*", ".*")
+            logger.debug(s"${if (matchesPattern) "retaining" else "omitting"} feature ${feature.id} with tileId $tileId")
+            matchesPattern
+          case _ =>
+            logger.warn(s"omitting feature ${feature.id} with unknown tileId")
+            false
+        })
+        case _ => features
+      }
   }
 
   case class STACCollection(id: String)

src/main/scala/org/openeo/opensearch/backends/CreodiasClient.scala

@@ -4,6 +4,7 @@ import geotrellis.proj4.LatLng
 import geotrellis.vector.{Extent, ProjectedExtent}
 import org.openeo.opensearch.OpenSearchClient
 import org.openeo.opensearch.OpenSearchResponses.{CreoCollections, CreoFeatureCollection, Feature, FeatureCollection}
+import org.slf4j.LoggerFactory
 import scalaj.http.HttpOptions
 
 import java.net.URL
@@ -12,6 +13,7 @@ import java.time.format.DateTimeFormatter.ISO_INSTANT
 import scala.collection.Map
 
 object CreodiasClient{
+  private val logger = LoggerFactory.getLogger(classOf[CreodiasClient])
 
   def apply(): CreodiasClient = {new CreodiasClient()}
 
@@ -74,7 +76,7 @@ class CreodiasClient(val endpoint: URL = new URL("https://catalogue.dataspace.co
       .param("maxRecords", "100")
       .param("status", "ONLINE")
       .param("dataset", "ESA-DATASET")
-      .params(attributeValues.mapValues(_.toString).filterKeys(!Seq( "eo:cloud_cover", "provider:backend", "orbitDirection", "sat:orbit_state", "processingBaseline").contains(_)).toSeq)
+      .params(attributeValues.mapValues(_.toString).filterKeys(isPropagated).toSeq)
 
     val cloudCover = attributeValues.get("eo:cloud_cover")
     if(cloudCover.isDefined) {
@@ -96,6 +98,15 @@ class CreodiasClient(val endpoint: URL = new URL("https://catalogue.dataspace.co
         .param("completionDate", dateRange.get._2 format ISO_INSTANT)
     }
 
+    val tileIdPattern = attributeValues.get("tileId").map(_.toString)
+
+    tileIdPattern match {
+      case Some(pattern) if !pattern.contains("*") =>
+        getProducts = getProducts.param("tileId", pattern)
+        logger.debug(s"included non-wildcard tileId $pattern param")
+      case _ =>
+    }
+
     /*
       // HACK: Putting pb as latest filter changes request time from 1.5min to 20sec.
       // Used this JS snippet to debug it:
@@ -111,9 +122,13 @@ class CreodiasClient(val endpoint: URL = new URL("https://catalogue.dataspace.co
     }
 
     val json = execute(getProducts)
-    CreoFeatureCollection.parse(json, dedup = true)
+    CreoFeatureCollection.parse(json, dedup = true, tileIdPattern)
   }
 
+  private def isPropagated(attribute: String): Boolean =
+    !Set("eo:cloud_cover", "provider:backend", "orbitDirection", "sat:orbit_state", "processingBaseline", "tileId")
+      .contains(attribute)
+
   override def getCollections(correlationId: String): Seq[Feature] = {
     val getCollections = http(s"$endpoint/collections.json")
       .option(HttpOptions.followRedirects(true))

src/test/resources/log4j.properties

@@ -3,7 +3,7 @@ log4j.appender.stdout=org.apache.log4j.ConsoleAppender
 log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
 log4j.appender.stdout.layout.ConversionPattern=%d{dd MMM HH:mm:ss} %p [%c{2}] - %m%n
 
-log4j.logger.org.openeo.geotrellissentinelhub=DEBUG
+log4j.logger.org.openeo=DEBUG
 log4j.logger.com.sksamuel.elastic4s=WARN
 log4j.logger.org.apache.zookeeper=WARN
 log4j.logger.org.apache.curator=WARN

src/test/resources/org/openeo/creoDiasTileIdPattern.json

@@ -0,0 +1 @@
+{"type":"FeatureCollection","properties":{"id":"727fa1c3-a58a-5c28-b6b8-eaec07a95eb1","totalResults":1,"exactCount":true,"startIndex":1,"itemsPerPage":1,"query":{"originalFilters":{"collection":"Sentinel2","box":"4.912740773449022,51.02800549763754,4.918234392041143,51.02998165805484","sortParam":"startDate","sortOrder":"ascending","page":"1","maxRecords":"100","productType":"L2A","tileId":"31UFS"},"appliedFilters":{"collection":"Sentinel2","box":"4.912740773449022,51.02800549763754,4.918234392041143,51.02998165805484","sortParam":"startDate","sortOrder":"ascending","page":"1","maxRecords":"100","productType":"L2A","tileId":"31UFS"},"processingTime":0.220491588},"links":[{"rel":"self","type":"application/json","title":"self","href":"https://catalogue.dataspace.copernicus.eu/resto/api/collections/Sentinel2/search.json?box=4.912740773449022%2C51.02800549763754%2C4.918234392041143%2C51.02998165805484&sortParam=startDate&sortOrder=ascending&page=1&maxRecords=100&status=ONLINE&dataset=ESA-DATASET&productType=L2A&startDate=2023-09-24T00%3A00%3A00Z&completionDate=2023-09-25T00%3A00%3A00Z&tileId=31UFS"},{"rel":"search","type":"application/opensearchdescription+xml","title":"OpenSearch Description Document","href":"https://catalogue.dataspace.copernicus.eu/resto/api/collections/Sentinel2/describe.xml"}]},"features":[{"type":"Feature","id":"48529695-77ec-4fbb-b0c8-2fbdf2e8bda4","geometry":{"type":"Polygon","coordinates":[[[4.43666142936438,51.3698516297421],[4.40871510963764,50.4552721606619],[5.95386971314071,50.4262936606196],[6.01702500852582,51.4123426550669],[4.46641328719322,51.4418286899677],[4.43666142936438,51.3698516297421]]]},"properties":{"collection":"SENTINEL-2","status":"ONLINE","license":{"licenseId":"unlicensed","hasToBeSigned":"never","grantedCountries":null,"grantedOrganizationCountries":null,"grantedFlags":null,"viewService":"public","signatureQuota":-1,"description":{"shortName":"No license"}},"productIdentifier":"/eodata/Sentinel-2/MSI/L2A/2023/09/24/S2B_MSIL2A_20230924T103659_N0509_R008_T31UFS_20230924T132849.SAFE","parentIdentifier":null,"title":"S2B_MSIL2A_20230924T103659_N0509_R008_T31UFS_20230924T132849.SAFE","description":"The Copernicus Sentinel-2 mission consists of two polar-orbiting satellites that are positioned in the same sun-synchronous orbit, with a phase difference of 180°. It aims to monitor changes in land surface conditions. The satellites have a wide swath width (290 km) and a high revisit time. Sentinel-2 is equipped with an optical instrument payload that samples 13 spectral bands: four bands at 10 m, six bands at 20 m and three bands at 60 m spatial resolution [https://dataspace.copernicus.eu/explore-data/data-collections/sentinel-data/sentinel-2].","organisationName":null,"startDate":"2023-09-24T10:36:59.024Z","completionDate":"2023-09-24T10:36:59.024Z","productType":"S2MSI2A","processingLevel":"S2MSI2A","platform":"S2B","instrument":"MSI","resolution":0,"sensorMode":null,"orbitNumber":34211,"quicklook":null,"thumbnail":"https://catalogue.dataspace.copernicus.eu/get-object?path=/Sentinel-2/MSI/L2A/2023/09/24/S2B_MSIL2A_20230924T103659_N0509_R008_T31UFS_20230924T132849.SAFE/S2B_MSIL2A_20230924T103659_N0509_R008_T31UFS_20230924T132849-ql.jpg","updated":"2023-09-24T16:01:48.906Z","published":"2023-09-24T16:01:39.303Z","snowCover":0,"cloudCover":10.342766,"gmlgeometry":"<gml:Polygon srsName=\"EPSG:4326\"><gml:outerBoundaryIs><gml:LinearRing><gml:coordinates>4.43666142936438,51.3698516297421 4.40871510963764,50.4552721606619 5.95386971314071,50.4262936606196 6.01702500852582,51.4123426550669 4.46641328719322,51.4418286899677 4.43666142936438,51.3698516297421</gml:coordinates></gml:LinearRing></gml:outerBoundaryIs></gml:Polygon>","centroid":{"type":"Point","coordinates":[5.20514092151565,50.9354916607594]},"orbitDirection":null,"timeliness":null,"relativeOrbitNumber":8,"processingBaseline":5.09,"missionTakeId":"GS2B_20230924T103659_034211_N05.09","services":{"download":{"url":"https://catalogue.dataspace.copernicus.eu/download/48529695-77ec-4fbb-b0c8-2fbdf2e8bda4","mimeType":"application/octet-stream","size":1204714187}},"links":[{"rel":"self","type":"application/json","title":"GeoJSON link for 48529695-77ec-4fbb-b0c8-2fbdf2e8bda4","href":"https://catalogue.dataspace.copernicus.eu/resto/collections/SENTINEL-2/48529695-77ec-4fbb-b0c8-2fbdf2e8bda4.json"}]}}]}

src/test/scala/org/openeo/CreodiasAPITest.scala

@@ -1,12 +1,14 @@
 package org.openeo
 
-import geotrellis.proj4.CRS
+import geotrellis.proj4.{CRS, LatLng}
 import geotrellis.vector.{Extent, ProjectedExtent}
+import org.junit.jupiter.api.Assertions.assertTrue
 import org.junit.jupiter.api.{AfterEach, BeforeEach, Test}
+import org.openeo.opensearch.OpenSearchResponses
 import org.openeo.opensearch.backends.CreodiasClient
 
 import java.time.ZoneOffset.UTC
-import java.time.ZonedDateTime
+import java.time.{LocalDate, ZonedDateTime}
 import scala.collection.Map
 
 class CreodiasAPITest {
@@ -38,8 +40,25 @@ class CreodiasAPITest {
     // The parser in getProducts should be able to handle these incorrect multipolygons.
     new CreodiasClient().getProducts(
       "Sentinel1", Some(fromDate, toDate),
-      bbox, attributeValues, "", ""
+      bbox, attributeValues
     )
   }
 
+  @Test
+  def testTileIdWithWildcard(): Unit = {
+    def getProducts(tileIdPattern: Option[String]): Seq[OpenSearchResponses.Feature] = {
+      new CreodiasClient().getProducts(
+        "Sentinel2",
+        dateRange = LocalDate.of(2023, 9, 24) -> LocalDate.of(2023, 9, 24),
+        bbox = ProjectedExtent(
+          Extent(4.912844218500582, 51.02816932187383, 4.918160603369832, 51.029815337603594), LatLng),
+        attributeValues = tileIdPattern.map("tileId" -> _).toMap,
+        correlationId = "",
+        processingLevel = ""
+      )
+    }
+
+    assertTrue(getProducts(tileIdPattern = None).nonEmpty) // sanity check
+    assertTrue(getProducts(tileIdPattern = Some("30*")).isEmpty)
+  }
 }

src/test/scala/org/openeo/OpenSearchResponsesTest.scala

@@ -4,26 +4,16 @@ import geotrellis.proj4.{CRS, LatLng}
 import geotrellis.vector.Extent
 import org.junit.Assert._
 import org.junit.Test
+import org.openeo.TestHelpers.loadJsonResource
 import org.openeo.opensearch.OpenSearchResponses
-import org.openeo.opensearch.OpenSearchResponses.{CreoFeatureCollection, FeatureCollection, STACFeatureCollection}
+import org.openeo.opensearch.OpenSearchResponses.{FeatureCollection, STACFeatureCollection}
 
 import java.io.{PrintWriter, StringWriter}
 import java.net.URI
 import java.time.ZonedDateTime
-import scala.io.{Codec, Source}
 
 class OpenSearchResponsesTest {
 
-  private val resourcePath = "/org/openeo/"
-
-  private def loadJsonResource(classPathResourceName: String, codec: Codec = Codec.UTF8): String = {
-    val fullPath = resourcePath + classPathResourceName
-    val jsonFile = Source.fromURL(getClass.getResource(fullPath))(codec)
-
-    try jsonFile.mkString
-    finally jsonFile.close()
-  }
-
   @Test
   def testReformat():Unit = {
     assertEquals("IMG_DATA_Band_B8A_20m_Tile1_Data",OpenSearchResponses.sentinel2Reformat("IMG_DATA_20m_Band9_Tile1_Data","GRANULE/L2A_T30SVH_A017537_20181031T110435/IMG_DATA/R20m/T30SVH_20181031T110201_B8A_20m.jp2"))
@@ -90,101 +80,6 @@ class OpenSearchResponsesTest {
     assertEquals(0, features.length)
   }
 
-  @Test
-  def parseCreodiasDuppedFeature(): Unit = {
-    val collectionsResponse = loadJsonResource("creodiasDuppedFeature.json")
-    val features = CreoFeatureCollection.parse(collectionsResponse, dedup = true).features
-
-    assertEquals(1, features.length)
-
-    val feature = features(0)
-
-    // Check if we really picked the latest Feature:
-    assertEquals(ZonedDateTime.parse("2021-04-11T08:51:07.054814Z"), feature.generalProperties.published.get)
-    assertEquals("/eodata/Sentinel-1/SAR/GRD/2021/04/11/S1B_IW_GRDH_1SDV_20210411T054146_20210411T054211_026415_032740_6184.SAFE", feature.id)
-  }
-
-  @Test
-  def parseCreodiasSpecialDupped(): Unit = {
-    val creodiasFeatureSnippet = loadJsonResource("creodiasFeatureSnippet.json")
-    val composedJsonString = """{
-      |  "type": "FeatureCollection",
-      |  "properties": {
-      |    "id": "fad6ca4a-3ab6-59e5-8e69-14aaf70256de",
-      |    "totalResults": 2,
-      |    "exactCount": true,
-      |    "startIndex": 1,
-      |    "itemsPerPage": 2,
-      |    "links": [
-      |      {
-      |        "rel": "self",
-      |        "type": "application/json",
-      |        "title": "self",
-      |        "href": "https://finder.creodias.eu/resto/api/collections/Sentinel1/search.json?&maxRecords=10&startDate=2021-04-11T00%3A00%3A00Z&completionDate=2021-04-11T23%3A59%3A59Z&productType=GRD&sensorMode=IW&geometry=POLYGON%28%285.785404630537803%2051.033953432779526%2C5.787426293119076%2051.021746940265956%2C5.803195261253003%2051.018694814851074%2C5.803195261253003%2051.02912208053834%2C5.785404630537803%2051.033953432779526%29%29&sortParam=startDate&sortOrder=descending&status=all&dataset=ESA-DATASET"
-      |      },
-      |      {
-      |        "rel": "search",
-      |        "type": "application/opensearchdescription+xml",
-      |        "title": "OpenSearch Description Document",
-      |        "href": "https://finder.creodias.eu/resto/api/collections/Sentinel1/describe.xml"
-      |      }
-      |    ]
-      |  },
-      |  "features": [""".stripMargin +
-      creodiasFeatureSnippet.replaceAll("%startDate%", "2000-01-01T01:01:01.001Z") + ", \n" +
-      creodiasFeatureSnippet.replaceAll("%startDate%", "2000-01-01T01:01:29.001Z") + ", \n" +
-      creodiasFeatureSnippet.replaceAll("%startDate%", "2000-01-01T01:01:32.001Z") +
-      "]}"
-    val features = CreoFeatureCollection.parse(composedJsonString, dedup = true).features
-
-    // Even tough the images are pairwise equal, the first and the last are not.
-    // So the dedup code is forced to consider them as separate
-    assertEquals(2, features.length)
-  }
-
-  @Test
-  def parseCreodiasDifferentGeom(): Unit = {
-    val collectionsResponse = loadJsonResource("creodiasDifferentGeom.json")
-    val features = CreoFeatureCollection.parse(collectionsResponse, dedup = true).features
-
-    assertEquals(3, features.length)
-  }
-
-  @Test
-  def parseCreodiasPhoebus(): Unit = {
-    val collectionsResponse = loadJsonResource("creodiasPhoebus.json")
-    val features = CreoFeatureCollection.parse(collectionsResponse, dedup = true).features
-
-    assertEquals(1, features.length)
-  }
-
-  @Test
-  def parseCreodiasMergePhoebusFeatures(): Unit = {
-    val collectionsResponse = loadJsonResource("creodiasMergePhoebusFeatures.json")
-    val features = CreoFeatureCollection.parse(collectionsResponse, dedup = true).features
-
-    assertEquals(1, features.length)
-  }
-
-  @Test
-  def creodiasOffsetNeeded(): Unit = {
-    val collectionsResponse = loadJsonResource("creodiasPixelValueOffsetNeeded.json")
-    val features = CreoFeatureCollection.parse(collectionsResponse, dedup = true).features
-    val link = features(0).links.find(l => l.title.get.contains("B04")).get
-    assertEquals(-1000, link.pixelValueOffset.get, 1e-6)
-
-    val linkSCL = features(0).links.find(l => l.title.get.contains("SCL")).get
-    assertEquals(0, linkSCL.pixelValueOffset.get, 1e-6)
-  }
-
-  @Test
-  def creodiasNoOffsetNeeded(): Unit = {
-    val collectionsResponse = loadJsonResource("creodiasDifferentGeom.json")
-    val features = CreoFeatureCollection.parse(collectionsResponse, dedup = true).features
-    val link = features(0).links.find(l => l.title.get.contains("B04")).get
-    assertEquals(0, link.pixelValueOffset.get, 1e-6)
-  }
-
   @Test
   def parseCollectionsResponse(): Unit = {
     val collectionsResponse = loadJsonResource("oscarsCollectionsResponse.json")
@@ -274,4 +169,4 @@ class OpenSearchResponsesTest {
 
     s.toString
   }
-}
+}

src/test/scala/org/openeo/TestHelpers.scala

@@ -0,0 +1,15 @@
+package org.openeo
+
+import scala.io.{Codec, Source}
+
+object TestHelpers {
+  private val resourcePath = "/org/openeo/"
+
+  private[openeo] def loadJsonResource(classPathResourceName: String, codec: Codec = Codec.UTF8): String = {
+    val fullPath = resourcePath + classPathResourceName
+    val jsonFile = Source.fromURL(getClass.getResource(fullPath))(codec)
+
+    try jsonFile.mkString
+    finally jsonFile.close()
+  }
+}