From d149049ae84a9e9e5bec5ccb244284b9b8a08e07 Mon Sep 17 00:00:00 2001 From: James Santucci Date: Wed, 12 May 2021 15:00:36 -0600 Subject: [PATCH 01/10] insert datetimes / time ranges with item inserts --- .../franklin/crawler/CatalogStacImport.scala | 2 +- .../franklin/crawler/FeatureExtractor.scala | 8 +++- .../franklin/database/StacItemDao.scala | 40 ++++++++++++------- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/application/src/main/scala/com/azavea/franklin/crawler/CatalogStacImport.scala b/application/src/main/scala/com/azavea/franklin/crawler/CatalogStacImport.scala index 7b8be310f..3d8e7de97 100644 --- a/application/src/main/scala/com/azavea/franklin/crawler/CatalogStacImport.scala +++ b/application/src/main/scala/com/azavea/franklin/crawler/CatalogStacImport.scala @@ -153,7 +153,7 @@ class CatalogStacImport(val catalogRoot: String) { ) ), ().asJsonObject, - forItem.properties, + forItem.properties.asJson.asObject.getOrElse(JsonObject.empty), List(parentCollectionLink, derivedFromItemLink), Some(Map.empty) ) diff --git a/application/src/main/scala/com/azavea/franklin/crawler/FeatureExtractor.scala b/application/src/main/scala/com/azavea/franklin/crawler/FeatureExtractor.scala index ed4a77ce5..f7a6189f5 100644 --- a/application/src/main/scala/com/azavea/franklin/crawler/FeatureExtractor.scala +++ b/application/src/main/scala/com/azavea/franklin/crawler/FeatureExtractor.scala @@ -6,6 +6,7 @@ import com.azavea.stac4s._ import geotrellis.vector.methods.Implicits._ import geotrellis.vector.{Feature, Geometry} import io.circe.JsonObject +import io.circe.syntax._ import java.net.URLEncoder import java.nio.charset.StandardCharsets @@ -44,7 +45,7 @@ object FeatureExtractor { StacItem( s"${UUID.randomUUID}", - "0.9.0", + "1.0.0-rc.2", Nil, "Feature", feature.geom, @@ -52,7 +53,10 @@ object FeatureExtractor { links = List(collectionLink, sourceItemLink), assets = Map.empty, collection = Some(inCollection.id), - properties = feature.data + properties = ItemProperties( + forItem.properties.datetime, + extensionFields = feature.data + ) ) } diff --git a/application/src/main/scala/com/azavea/franklin/database/StacItemDao.scala b/application/src/main/scala/com/azavea/franklin/database/StacItemDao.scala index 125932b95..4162c575d 100644 --- a/application/src/main/scala/com/azavea/franklin/database/StacItemDao.scala +++ b/application/src/main/scala/com/azavea/franklin/database/StacItemDao.scala @@ -4,11 +4,8 @@ import cats.data.EitherT import cats.data.NonEmptyList import cats.data.OptionT import cats.syntax.all._ -import com.azavea.franklin.datamodel.BulkExtent -import com.azavea.franklin.datamodel.Context import com.azavea.franklin.datamodel.PaginationToken import com.azavea.franklin.datamodel.SearchMethod -import com.azavea.franklin.datamodel.StacSearchCollection import com.azavea.franklin.extensions.paging.PagingLinkExtension import com.azavea.stac4s._ import com.azavea.stac4s.extensions.periodic.PeriodicExtent @@ -17,7 +14,6 @@ import com.azavea.stac4s.syntax._ import doobie.Fragment import doobie.free.connection.ConnectionIO import doobie.implicits._ -import doobie.implicits.javatime._ import doobie.refined.implicits._ import doobie.util.update.Update import eu.timepit.refined.auto._ @@ -33,7 +29,6 @@ import org.threeten.extra.PeriodDuration import java.time.Instant import java.time.LocalDateTime -import java.time.Period import java.time.ZoneId import java.time.ZoneOffset @@ -230,17 +225,21 @@ object StacItemDao extends Dao[StacItem] { id: String, geom: Projected[Geometry], item: StacItem, - collection: Option[String] + collection: Option[String], + startDatetime: Option[Instant], + endDatetime: Option[Instant], + datetime: Option[Instant] ) def insertManyStacItems( items: List[StacItem], collection: StacCollection ): ConnectionIO[(Set[String], Int)] = { - val insertFragment = """ - INSERT INTO collection_items (id, geom, item, collection) + val insertFragment = + """ + INSERT INTO collection_items (id, geom, item, collection, start_datetime, end_datetime, datetime) VALUES - (?, ?, ?, ?) + (?, ?, ?, ?, ?, ?, ?) """ val badIds = items .mapFilter(item => @@ -252,7 +251,13 @@ object StacItemDao extends Dao[StacItem] { val stacItemInserts = items .filter(item => (!badIds.contains(item.id))) - .map(i => StacItemBulkImport(i.id, Projected(i.geometry, 4326), i, i.collection)) + .map(i => { + val timeRangeO = StacItem.timeRangePrism.getOption(i) + val datetimeO = StacItem.datetimePrism.getOption(i) + StacItemBulkImport(i.id, Projected(i.geometry, 4326), i, i.collection, timeRangeO map { + _.start + }, timeRangeO map { _.end }, datetimeO map { _.when }) + }) Update[StacItemBulkImport](insertFragment).updateMany(stacItemInserts) map { numberInserted => (badIds, numberInserted) } @@ -262,10 +267,15 @@ object StacItemDao extends Dao[StacItem] { val projectedGeometry = Projected(item.geometry, 4326) + val timeRangeO = StacItem.timeRangePrism.getOption(item) + val startO = timeRangeO map { _.start } + val endO = timeRangeO map { _.end } + val datetimeO = StacItem.datetimePrism.getOption(item) map { _.when } + val insertFragment = fr""" - INSERT INTO collection_items (id, geom, item, collection) + INSERT INTO collection_items (id, geom, item, collection, start_datetime, end_datetime, datetime) VALUES - (${item.id}, $projectedGeometry, $item, ${item.collection}) + (${item.id}, $projectedGeometry, $item, ${item.collection}, ${startO}, ${endO}, ${datetimeO}) """ for { collectionE <- (item.collection.flatTraverse(collectionId => @@ -365,9 +375,9 @@ object StacItemDao extends Dao[StacItem] { case (Right(patchedItem), true) => checkItemTimeAgainstCollection(collectionInDb, patchedItem) .leftWiden[StacItemDaoError] flatTraverse { validated => - doUpdate(itemId, validated.copy(properties = patchedItem.properties.filter({ - case (_, v) => !v.isNull - }))).attempt map { _.leftMap(_ => UpdateFailed: StacItemDaoError) } + doUpdate(itemId, validated).attempt map { + _.leftMap(_ => UpdateFailed: StacItemDaoError) + } } case (_, false) => (Either.left[StacItemDaoError, StacItem](StaleObject)).pure[ConnectionIO] From 8efa09f14d2151a8afd175b852e6ad1da71790e0 Mon Sep 17 00:00:00 2001 From: James Santucci Date: Wed, 12 May 2021 15:05:40 -0600 Subject: [PATCH 02/10] factor out time access and update in updates --- .../franklin/database/StacItemDao.scala | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/application/src/main/scala/com/azavea/franklin/database/StacItemDao.scala b/application/src/main/scala/com/azavea/franklin/database/StacItemDao.scala index 4162c575d..e37e0a166 100644 --- a/application/src/main/scala/com/azavea/franklin/database/StacItemDao.scala +++ b/application/src/main/scala/com/azavea/franklin/database/StacItemDao.scala @@ -134,6 +134,14 @@ object StacItemDao extends Dao[StacItem] { ) } + private def getTimeData(item: StacItem): (Option[Instant], Option[Instant], Option[Instant]) = { + val timeRangeO = StacItem.timeRangePrism.getOption(item) + val startO = timeRangeO map { _.start } + val endO = timeRangeO map { _.end } + val datetimeO = StacItem.datetimePrism.getOption(item) map { _.when } + (startO, endO, datetimeO) + } + def getItemCount(): ConnectionIO[Int] = { sql"select count(*) from collection_items".query[Int].unique } @@ -252,11 +260,16 @@ object StacItemDao extends Dao[StacItem] { items .filter(item => (!badIds.contains(item.id))) .map(i => { - val timeRangeO = StacItem.timeRangePrism.getOption(i) - val datetimeO = StacItem.datetimePrism.getOption(i) - StacItemBulkImport(i.id, Projected(i.geometry, 4326), i, i.collection, timeRangeO map { - _.start - }, timeRangeO map { _.end }, datetimeO map { _.when }) + val (startO, endO, datetimeO) = getTimeData(i) + StacItemBulkImport( + i.id, + Projected(i.geometry, 4326), + i, + i.collection, + startO, + endO, + datetimeO + ) }) Update[StacItemBulkImport](insertFragment).updateMany(stacItemInserts) map { numberInserted => (badIds, numberInserted) @@ -267,10 +280,7 @@ object StacItemDao extends Dao[StacItem] { val projectedGeometry = Projected(item.geometry, 4326) - val timeRangeO = StacItem.timeRangePrism.getOption(item) - val startO = timeRangeO map { _.start } - val endO = timeRangeO map { _.end } - val datetimeO = StacItem.datetimePrism.getOption(item) map { _.when } + val (startO, endO, datetimeO) = getTimeData(item) val insertFragment = fr""" INSERT INTO collection_items (id, geom, item, collection, start_datetime, end_datetime, datetime) @@ -309,10 +319,14 @@ object StacItemDao extends Dao[StacItem] { .selectOption private def doUpdate(itemId: String, item: StacItem): ConnectionIO[StacItem] = { - val fragment = fr""" + val (startO, endO, datetimeO) = getTimeData(item) + val fragment = fr""" UPDATE collection_items SET - item = $item + item = $item, + start_datetime = $startO, + end_datetime = $endO, + datetime = $datetimeO WHERE id = $itemId """ fragment.update.withUniqueGeneratedKeys[StacItem]("item") From c4ee9014b2df2e5607ea356493846e5c4ef46dce Mon Sep 17 00:00:00 2001 From: James Santucci Date: Wed, 12 May 2021 15:18:21 -0600 Subject: [PATCH 03/10] update tests for always having an item datetime --- .../franklin/api/services/FiltersFor.scala | 69 ++++++++++--------- .../api/services/SearchServiceSpec.scala | 51 +++++++------- 2 files changed, 60 insertions(+), 60 deletions(-) diff --git a/application/src/test/scala/com/azavea/franklin/api/services/FiltersFor.scala b/application/src/test/scala/com/azavea/franklin/api/services/FiltersFor.scala index 72790a970..de5cc953e 100644 --- a/application/src/test/scala/com/azavea/franklin/api/services/FiltersFor.scala +++ b/application/src/test/scala/com/azavea/franklin/api/services/FiltersFor.scala @@ -5,17 +5,14 @@ import cats.data.NonEmptyList import cats.syntax.all._ import com.azavea.franklin.database.SearchFilters import com.azavea.stac4s.jvmTypes.TemporalExtent -import com.azavea.stac4s.{StacCollection, StacItem, TwoDimBbox} +import com.azavea.stac4s.{ItemDatetime, StacCollection, StacItem, TwoDimBbox} import geotrellis.vector.Extent -import io.circe.optics._ import io.circe.syntax._ import java.time.Instant object FiltersFor { - val datetimePrism = JsonPath.root.datetime.as[Instant] - implicit val searchFilterSemigroup: Semigroup[SearchFilters] = new Semigroup[SearchFilters] { def combine(x: SearchFilters, y: SearchFilters): SearchFilters = SearchFilters( @@ -51,21 +48,24 @@ object FiltersFor { ) } - def timeFilterFor(item: StacItem): Option[SearchFilters] = - datetimePrism.getOption(item.properties.asJson) map { instant => - TemporalExtent(instant.minusSeconds(60), Some(instant.plusSeconds(60))) - } map { temporalExtent => - SearchFilters( - None, - Some(temporalExtent), - None, - Nil, - Nil, - None, - Map.empty, - None - ) + def timeFilterFor(item: StacItem): SearchFilters = { + val temporalExtent = item.properties.datetime match { + case ItemDatetime.PointInTime(instant) => + TemporalExtent(instant.minusSeconds(60), Some(instant.plusSeconds(60))) + case ItemDatetime.TimeRange(start, end) => + TemporalExtent(start.minusSeconds(60), Some(end.plusSeconds(60))) } + SearchFilters( + None, + Some(temporalExtent), + None, + Nil, + Nil, + None, + Map.empty, + None + ) + } def geomFilterFor(item: StacItem): SearchFilters = SearchFilters( None, @@ -118,21 +118,24 @@ object FiltersFor { ) } - def timeFilterExcluding(item: StacItem): Option[SearchFilters] = - datetimePrism.getOption(item.properties.asJson) map { instant => - TemporalExtent(instant.minusSeconds(60), Some(instant.minusSeconds(30))) - } map { temporalExtent => - SearchFilters( - None, - Some(temporalExtent), - None, - Nil, - Nil, - None, - Map.empty, - None - ) + def timeFilterExcluding(item: StacItem): SearchFilters = { + val temporalExtent = item.properties.datetime match { + case ItemDatetime.PointInTime(instant) => + TemporalExtent(instant.minusSeconds(60), Some(instant.minusSeconds(30))) + case ItemDatetime.TimeRange(start, _) => + TemporalExtent(start.minusSeconds(60), Some(start.minusSeconds(30))) } + SearchFilters( + None, + Some(temporalExtent), + None, + Nil, + Nil, + None, + Map.empty, + None + ) + } def geomFilterExcluding(item: StacItem): SearchFilters = { val itemGeomBbox = item.geometry.getEnvelopeInternal() @@ -179,7 +182,7 @@ object FiltersFor { val filters: NonEmptyList[Option[SearchFilters]] = NonEmptyList .of( bboxFilterFor(item).some, - timeFilterFor(item), + timeFilterFor(item).some, geomFilterFor(item).some, collectionFilterFor(collection).some, itemFilterFor(item).some diff --git a/application/src/test/scala/com/azavea/franklin/api/services/SearchServiceSpec.scala b/application/src/test/scala/com/azavea/franklin/api/services/SearchServiceSpec.scala index d68afcb7f..aa51d3b7c 100644 --- a/application/src/test/scala/com/azavea/franklin/api/services/SearchServiceSpec.scala +++ b/application/src/test/scala/com/azavea/franklin/api/services/SearchServiceSpec.scala @@ -41,36 +41,33 @@ class SearchServiceSpec private def getExclusionTest( name: String - )(getFilters: StacCollection => StacItem => Option[SearchFilters]) = + )(getFilters: StacCollection => StacItem => SearchFilters) = prop { (stacItem: StacItem, stacCollection: StacCollection) => - val exclusiveParams = getFilters(stacCollection)(stacItem) - val testResult = exclusiveParams.map({ params => + val params = getFilters(stacCollection)(stacItem) + val testResult = { val resourceIO = testClient map { _.getCollectionItemResource(stacItem, stacCollection) } - val requestIO = resourceIO flatMap { - resource => - resource.use { - case _ => - // doing this as a POST is important, since otherwise the `intersection` and - // `query` params would be ignored (not that we're testing `query` here) - val request = - Request[IO]( - method = Method.POST, - uri = Uri.unsafeFromString(s"/search") - ).withEntity(params) - (for { - resp <- testServices.searchService.routes.run(request) - decoded <- OptionT.liftF { resp.as[StacSearchCollection] } - } yield decoded).value - } + val requestIO = resourceIO flatMap { resource => + resource.use { + case _ => + // doing this as a POST is important, since otherwise the `intersection` and + // `query` params would be ignored (not that we're testing `query` here) + val request = + Request[IO]( + method = Method.POST, + uri = Uri.unsafeFromString(s"/search") + ).withEntity(params) + (for { + resp <- testServices.searchService.routes.run(request) + decoded <- OptionT.liftF { resp.as[StacSearchCollection] } + } yield decoded).value + } } val result = requestIO.unsafeRunSync.get result.features.map(_.id).contains(stacItem.id) - }) + } - (testResult must beSome(false)) or (testResult must beNone and skipped( - s"$name did not produce filters" - )) + testResult aka s"the item was included in the results for $name" must beFalse } def postSearchFiltersExpectation = prop { (searchFilters: SearchFilters) => @@ -121,16 +118,16 @@ class SearchServiceSpec getExclusionTest("temporal extent")(_ => item => FiltersFor.timeFilterExcluding(item)) def dontFindBboxFilters = - getExclusionTest("bbox")(_ => item => FiltersFor.bboxFilterExcluding(item).some) + getExclusionTest("bbox")(_ => item => FiltersFor.bboxFilterExcluding(item)) def dontFindGeomFilters = - getExclusionTest("geom intersection")(_ => item => FiltersFor.geomFilterExcluding(item).some) + getExclusionTest("geom intersection")(_ => item => FiltersFor.geomFilterExcluding(item)) def dontFindCollectionFilters = getExclusionTest("collection ids")(collection => - _ => FiltersFor.collectionFilterExcluding(collection).some + _ => FiltersFor.collectionFilterExcluding(collection) ) def dontFindItemFilters = - getExclusionTest("item ids")(_ => item => FiltersFor.itemFilterExcluding(item).some) + getExclusionTest("item ids")(_ => item => FiltersFor.itemFilterExcluding(item)) } From 162a07b9942183cf3a11be909259ae3d0d8d9a43 Mon Sep 17 00:00:00 2001 From: James Santucci Date: Wed, 12 May 2021 15:26:13 -0600 Subject: [PATCH 04/10] simplify filter generation a little more --- .../franklin/api/services/FiltersFor.scala | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/application/src/test/scala/com/azavea/franklin/api/services/FiltersFor.scala b/application/src/test/scala/com/azavea/franklin/api/services/FiltersFor.scala index de5cc953e..862f88e2c 100644 --- a/application/src/test/scala/com/azavea/franklin/api/services/FiltersFor.scala +++ b/application/src/test/scala/com/azavea/franklin/api/services/FiltersFor.scala @@ -1,6 +1,6 @@ package com.azavea.franklin.api.services -import cats.Semigroup +import cats.{Monoid, Semigroup} import cats.data.NonEmptyList import cats.syntax.all._ import com.azavea.franklin.database.SearchFilters @@ -13,7 +13,7 @@ import java.time.Instant object FiltersFor { - implicit val searchFilterSemigroup: Semigroup[SearchFilters] = new Semigroup[SearchFilters] { + implicit val searchFilterMonoid: Monoid[SearchFilters] = new Monoid[SearchFilters] { def combine(x: SearchFilters, y: SearchFilters): SearchFilters = SearchFilters( x.bbox orElse y.bbox, @@ -25,6 +25,17 @@ object FiltersFor { x.query |+| y.query, x.next orElse y.next ) + + def empty: SearchFilters = SearchFilters( + None, + None, + None, + List.empty, + List.empty, + None, + Map.empty, + None + ) } def bboxFilterFor(item: StacItem): SearchFilters = { @@ -179,17 +190,14 @@ object FiltersFor { ) def inclusiveFilters(collection: StacCollection, item: StacItem): SearchFilters = { - val filters: NonEmptyList[Option[SearchFilters]] = NonEmptyList + val filters: NonEmptyList[SearchFilters] = NonEmptyList .of( - bboxFilterFor(item).some, - timeFilterFor(item).some, - geomFilterFor(item).some, - collectionFilterFor(collection).some, - itemFilterFor(item).some + bboxFilterFor(item), + timeFilterFor(item), + geomFilterFor(item), + collectionFilterFor(collection), + itemFilterFor(item) ) - val concatenated = filters.combineAll - // guaranteed to succeed, since most of the filters are being converted into options - // just to cooperate with timeFilterFor - concatenated.get + filters.combineAll } } From 8fc7329a72300057529e974531fa591a8f94f468 Mon Sep 17 00:00:00 2001 From: James Santucci Date: Wed, 12 May 2021 16:02:50 -0600 Subject: [PATCH 05/10] correct time filtering behavior to respect ranges --- .../scala/com/azavea/franklin/database/Filterables.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/application/src/main/scala/com/azavea/franklin/database/Filterables.scala b/application/src/main/scala/com/azavea/franklin/database/Filterables.scala index edfe689cc..1ff127d9f 100644 --- a/application/src/main/scala/com/azavea/franklin/database/Filterables.scala +++ b/application/src/main/scala/com/azavea/franklin/database/Filterables.scala @@ -18,11 +18,13 @@ trait FilterHelpers { def toFilterFragment: Option[Fragment] = { temporalExtent.value match { case Some(start) :: Some(end) :: _ => - Some(fr"(datetime >= $start AND datetime <= $end)") + Some( + fr"(datetime >= $start AND datetime <= $end) OR (start_datetime >= $start AND end_datetime <= $end)" + ) case Some(start) :: _ => - Some(fr"datetime >= $start") + Some(fr"(datetime >= $start OR start_datetime >= $start)") case _ :: Some(end) :: _ => - Some(fr"datetime <= $end") + Some(fr"(datetime <= $end OR end_datetime <= $end)") case _ => None } } From 6a95ecb1c81f6e7e623a5fcb9a2d45490b8aec44 Mon Sep 17 00:00:00 2001 From: James Santucci Date: Wed, 12 May 2021 16:02:58 -0600 Subject: [PATCH 06/10] do a much more thorough job testing time range filters --- .../franklin/api/services/FiltersFor.scala | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/application/src/test/scala/com/azavea/franklin/api/services/FiltersFor.scala b/application/src/test/scala/com/azavea/franklin/api/services/FiltersFor.scala index 862f88e2c..e2aff06cd 100644 --- a/application/src/test/scala/com/azavea/franklin/api/services/FiltersFor.scala +++ b/application/src/test/scala/com/azavea/franklin/api/services/FiltersFor.scala @@ -64,7 +64,17 @@ object FiltersFor { case ItemDatetime.PointInTime(instant) => TemporalExtent(instant.minusSeconds(60), Some(instant.plusSeconds(60))) case ItemDatetime.TimeRange(start, end) => - TemporalExtent(start.minusSeconds(60), Some(end.plusSeconds(60))) + val milli = start.toEpochMilli % 3 + if (milli == 0) { + // test start and end with full overlap + TemporalExtent(start.minusSeconds(60), Some(end.plusSeconds(60))) + } else if (milli == 1) { + // test start before the range start with open end + TemporalExtent(start.minusSeconds(60), None) + } else { + // test end after the range end with open start + TemporalExtent(None, end.plusSeconds(60)) + } } SearchFilters( None, @@ -133,8 +143,18 @@ object FiltersFor { val temporalExtent = item.properties.datetime match { case ItemDatetime.PointInTime(instant) => TemporalExtent(instant.minusSeconds(60), Some(instant.minusSeconds(30))) - case ItemDatetime.TimeRange(start, _) => - TemporalExtent(start.minusSeconds(60), Some(start.minusSeconds(30))) + case ItemDatetime.TimeRange(start, end) => + val milli = start.toEpochMilli % 3 + if (milli == 0) { + // test no intersection with range + TemporalExtent(start.minusSeconds(60), Some(start.minusSeconds(30))) + } else if (milli == 1) { + // test start after the range end with open end + TemporalExtent(end.plusSeconds(60), None) + } else { + // test end before the range start with open start + TemporalExtent(None, start.minusSeconds(60)) + } } SearchFilters( None, From ea61c5986b2f89f656aa1d54d0af3f6e5e55c328 Mon Sep 17 00:00:00 2001 From: James Santucci Date: Wed, 12 May 2021 16:04:29 -0600 Subject: [PATCH 07/10] bump to stac4s version supporting time improvements --- project/Versions.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/Versions.scala b/project/Versions.scala index 78acaf2a7..3d559eade 100644 --- a/project/Versions.scala +++ b/project/Versions.scala @@ -36,7 +36,7 @@ object Versions { val Slf4jVersion = "1.7.30" val Specs2Version = "4.11.0" val SpireVersion = "0.13.0" - val Stac4SVersion = "0.3.0" + val Stac4SVersion = "0.4.0" val SttpClientVersion = "2.2.9" val SttpShared = "1.1.1" val SttpModelVersion = "1.4.6" From 37fb1aeb6d5bfac97e15a279ee51a962eb574407 Mon Sep 17 00:00:00 2001 From: James Santucci Date: Wed, 12 May 2021 16:39:43 -0600 Subject: [PATCH 08/10] test all properties update now that that's more interesting --- .../franklin/api/services/CollectionItemsServiceSpec.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/application/src/test/scala/com/azavea/franklin/api/services/CollectionItemsServiceSpec.scala b/application/src/test/scala/com/azavea/franklin/api/services/CollectionItemsServiceSpec.scala index a3458911b..c2787a5d9 100644 --- a/application/src/test/scala/com/azavea/franklin/api/services/CollectionItemsServiceSpec.scala +++ b/application/src/test/scala/com/azavea/franklin/api/services/CollectionItemsServiceSpec.scala @@ -170,10 +170,15 @@ class CollectionItemsServiceSpec val sourceAssetsWithoutNulls = update.assets.asJson + val updateProperties = update.properties + + val updatedProperties = updated.properties + (updated.stacExtensions should beTypedEqualTo(update.stacExtensions)) and (resultAssetsWithoutNulls should beTypedEqualTo(sourceAssetsWithoutNulls)) and (updated.geometry should beTypedEqualTo(update.geometry)) and - (updated.bbox should beTypedEqualTo(update.bbox)) + (updated.bbox should beTypedEqualTo(update.bbox)) and + (updatedProperties should beTypedEqualTo(updateProperties)) } def patchItemExpectation = prop { (stacCollection: StacCollection, stacItem: StacItem) => From 35fdd4055161b3cb9edb8901f3717f3e8604bd25 Mon Sep 17 00:00:00 2001 From: James Santucci Date: Wed, 12 May 2021 17:02:34 -0600 Subject: [PATCH 09/10] add migration to fix existing items --- .../migrations/V12__update_time_fields_again.sql | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 application/src/main/resources/migrations/V12__update_time_fields_again.sql diff --git a/application/src/main/resources/migrations/V12__update_time_fields_again.sql b/application/src/main/resources/migrations/V12__update_time_fields_again.sql new file mode 100644 index 000000000..39dbc9d92 --- /dev/null +++ b/application/src/main/resources/migrations/V12__update_time_fields_again.sql @@ -0,0 +1,16 @@ +UPDATE + collection_items +SET + datetime = (item -> 'properties' ->> 'datetime') :: timestamp with time zone +WHERE + item -> 'properties' ? 'datetime' + AND datetime is null; + +UPDATE + collection_items +SET + start_datetime = (item -> 'properties' ->> 'start_datetime') :: timestamp with time zone, + end_datetime = (item -> 'properties' ->> 'end_datetime') :: timestamp with time zone +WHERE + item -> 'properties' ? 'start_datetime' + AND start_datetime is null; \ No newline at end of file From 77a7fc63abbe9c2f12eaeeb33ec7d679ba670558 Mon Sep 17 00:00:00 2001 From: James Santucci Date: Wed, 12 May 2021 17:17:48 -0600 Subject: [PATCH 10/10] lint --- .../scala/com/azavea/franklin/api/services/FiltersFor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/test/scala/com/azavea/franklin/api/services/FiltersFor.scala b/application/src/test/scala/com/azavea/franklin/api/services/FiltersFor.scala index e2aff06cd..121034dc8 100644 --- a/application/src/test/scala/com/azavea/franklin/api/services/FiltersFor.scala +++ b/application/src/test/scala/com/azavea/franklin/api/services/FiltersFor.scala @@ -1,8 +1,8 @@ package com.azavea.franklin.api.services -import cats.{Monoid, Semigroup} import cats.data.NonEmptyList import cats.syntax.all._ +import cats.{Monoid, Semigroup} import com.azavea.franklin.database.SearchFilters import com.azavea.stac4s.jvmTypes.TemporalExtent import com.azavea.stac4s.{ItemDatetime, StacCollection, StacItem, TwoDimBbox}