Do a much more thorough job with item times

application/src/main/resources/migrations/V12__update_time_fields_again.sql

@@ -0,0 +1,16 @@
+UPDATE
+    collection_items
+SET
+    datetime = (item -> 'properties' ->> 'datetime') :: timestamp with time zone
+WHERE
+    item -> 'properties' ? 'datetime'
+    AND datetime is null;
+
+UPDATE
+    collection_items
+SET
+    start_datetime = (item -> 'properties' ->> 'start_datetime') :: timestamp with time zone,
+    end_datetime = (item -> 'properties' ->> 'end_datetime') :: timestamp with time zone
+WHERE
+    item -> 'properties' ? 'start_datetime'
+    AND start_datetime is null;

application/src/main/scala/com/azavea/franklin/crawler/CatalogStacImport.scala

@@ -153,7 +153,7 @@ class CatalogStacImport(val catalogRoot: String) {
                       )
                     ),
                     ().asJsonObject,
-                    forItem.properties,
+                    forItem.properties.asJson.asObject.getOrElse(JsonObject.empty),
                     List(parentCollectionLink, derivedFromItemLink),
                     Some(Map.empty)
                   )

application/src/main/scala/com/azavea/franklin/crawler/FeatureExtractor.scala

@@ -6,6 +6,7 @@ import com.azavea.stac4s._
 import geotrellis.vector.methods.Implicits._
 import geotrellis.vector.{Feature, Geometry}
 import io.circe.JsonObject
+import io.circe.syntax._
 
 import java.net.URLEncoder
 import java.nio.charset.StandardCharsets
@@ -44,15 +45,18 @@ object FeatureExtractor {
 
     StacItem(
       s"${UUID.randomUUID}",
-      "0.9.0",
+      "1.0.0-rc.2",
       Nil,
       "Feature",
       feature.geom,
       TwoDimBbox(featureExtent.xmin, featureExtent.ymin, featureExtent.xmax, featureExtent.ymax),
       links = List(collectionLink, sourceItemLink),
       assets = Map.empty,
       collection = Some(inCollection.id),
-      properties = feature.data
+      properties = ItemProperties(
+        forItem.properties.datetime,
+        extensionFields = feature.data
+      )
     )
   }
 

application/src/main/scala/com/azavea/franklin/database/Filterables.scala

@@ -18,11 +18,13 @@ trait FilterHelpers {
     def toFilterFragment: Option[Fragment] = {
       temporalExtent.value match {
         case Some(start) :: Some(end) :: _ =>
-          Some(fr"(datetime >= $start AND datetime <= $end)")
+          Some(
+            fr"(datetime >= $start AND datetime <= $end) OR (start_datetime >= $start AND end_datetime <= $end)"
+          )
         case Some(start) :: _ =>
-          Some(fr"datetime >= $start")
+          Some(fr"(datetime >= $start OR start_datetime >= $start)")
         case _ :: Some(end) :: _ =>
-          Some(fr"datetime <= $end")
+          Some(fr"(datetime <= $end OR end_datetime <= $end)")
         case _ => None
       }
     }

application/src/main/scala/com/azavea/franklin/database/StacItemDao.scala

@@ -4,11 +4,8 @@ import cats.data.EitherT
 import cats.data.NonEmptyList
 import cats.data.OptionT
 import cats.syntax.all._
-import com.azavea.franklin.datamodel.BulkExtent
-import com.azavea.franklin.datamodel.Context
 import com.azavea.franklin.datamodel.PaginationToken
 import com.azavea.franklin.datamodel.SearchMethod
-import com.azavea.franklin.datamodel.StacSearchCollection
 import com.azavea.franklin.extensions.paging.PagingLinkExtension
 import com.azavea.stac4s._
 import com.azavea.stac4s.extensions.periodic.PeriodicExtent
@@ -17,7 +14,6 @@ import com.azavea.stac4s.syntax._
 import doobie.Fragment
 import doobie.free.connection.ConnectionIO
 import doobie.implicits._
-import doobie.implicits.javatime._
 import doobie.refined.implicits._
 import doobie.util.update.Update
 import eu.timepit.refined.auto._
@@ -33,7 +29,6 @@ import org.threeten.extra.PeriodDuration
 
 import java.time.Instant
 import java.time.LocalDateTime
-import java.time.Period
 import java.time.ZoneId
 import java.time.ZoneOffset
 
@@ -139,6 +134,14 @@ object StacItemDao extends Dao[StacItem] {
       )
   }
 
+  private def getTimeData(item: StacItem): (Option[Instant], Option[Instant], Option[Instant]) = {
+    val timeRangeO = StacItem.timeRangePrism.getOption(item)
+    val startO     = timeRangeO map { _.start }
+    val endO       = timeRangeO map { _.end }
+    val datetimeO  = StacItem.datetimePrism.getOption(item) map { _.when }
+    (startO, endO, datetimeO)
+  }
+
   def getItemCount(): ConnectionIO[Int] = {
     sql"select count(*) from collection_items".query[Int].unique
   }
@@ -230,17 +233,21 @@ object StacItemDao extends Dao[StacItem] {
       id: String,
       geom: Projected[Geometry],
       item: StacItem,
-      collection: Option[String]
+      collection: Option[String],
+      startDatetime: Option[Instant],
+      endDatetime: Option[Instant],
+      datetime: Option[Instant]
   )
 
   def insertManyStacItems(
       items: List[StacItem],
       collection: StacCollection
   ): ConnectionIO[(Set[String], Int)] = {
-    val insertFragment = """
-      INSERT INTO collection_items (id, geom, item, collection)
+    val insertFragment =
+      """
+      INSERT INTO collection_items (id, geom, item, collection, start_datetime, end_datetime, datetime)
       VALUES
-      (?, ?, ?, ?)
+      (?, ?, ?, ?, ?, ?, ?)
       """
     val badIds = items
       .mapFilter(item =>
@@ -252,7 +259,18 @@ object StacItemDao extends Dao[StacItem] {
     val stacItemInserts =
       items
         .filter(item => (!badIds.contains(item.id)))
-        .map(i => StacItemBulkImport(i.id, Projected(i.geometry, 4326), i, i.collection))
+        .map(i => {
+          val (startO, endO, datetimeO) = getTimeData(i)
+          StacItemBulkImport(
+            i.id,
+            Projected(i.geometry, 4326),
+            i,
+            i.collection,
+            startO,
+            endO,
+            datetimeO
+          )
+        })
     Update[StacItemBulkImport](insertFragment).updateMany(stacItemInserts) map { numberInserted =>
       (badIds, numberInserted)
     }
@@ -262,10 +280,12 @@ object StacItemDao extends Dao[StacItem] {
 
     val projectedGeometry = Projected(item.geometry, 4326)
 
+    val (startO, endO, datetimeO) = getTimeData(item)
+
     val insertFragment = fr"""
-      INSERT INTO collection_items (id, geom, item, collection)
+      INSERT INTO collection_items (id, geom, item, collection, start_datetime, end_datetime, datetime)
       VALUES
-      (${item.id}, $projectedGeometry, $item, ${item.collection})
+      (${item.id}, $projectedGeometry, $item, ${item.collection}, ${startO}, ${endO}, ${datetimeO})
       """
     for {
       collectionE <- (item.collection.flatTraverse(collectionId =>
@@ -299,10 +319,14 @@ object StacItemDao extends Dao[StacItem] {
       .selectOption
 
   private def doUpdate(itemId: String, item: StacItem): ConnectionIO[StacItem] = {
-    val fragment = fr"""
+    val (startO, endO, datetimeO) = getTimeData(item)
+    val fragment                  = fr"""
       UPDATE collection_items
       SET
-        item = $item
+        item = $item,
+        start_datetime = $startO,
+        end_datetime = $endO,
+        datetime = $datetimeO
       WHERE id = $itemId
     """
     fragment.update.withUniqueGeneratedKeys[StacItem]("item")
@@ -365,9 +389,9 @@ object StacItemDao extends Dao[StacItem] {
             case (Right(patchedItem), true) =>
               checkItemTimeAgainstCollection(collectionInDb, patchedItem)
                 .leftWiden[StacItemDaoError] flatTraverse { validated =>
-                doUpdate(itemId, validated.copy(properties = patchedItem.properties.filter({
-                  case (_, v) => !v.isNull
-                }))).attempt map { _.leftMap(_ => UpdateFailed: StacItemDaoError) }
+                doUpdate(itemId, validated).attempt map {
+                  _.leftMap(_ => UpdateFailed: StacItemDaoError)
+                }
               }
             case (_, false) =>
               (Either.left[StacItemDaoError, StacItem](StaleObject)).pure[ConnectionIO]

application/src/test/scala/com/azavea/franklin/api/services/CollectionItemsServiceSpec.scala

@@ -170,10 +170,15 @@ class CollectionItemsServiceSpec
 
       val sourceAssetsWithoutNulls = update.assets.asJson
 
+      val updateProperties = update.properties
+
+      val updatedProperties = updated.properties
+
       (updated.stacExtensions should beTypedEqualTo(update.stacExtensions)) and
         (resultAssetsWithoutNulls should beTypedEqualTo(sourceAssetsWithoutNulls)) and
         (updated.geometry should beTypedEqualTo(update.geometry)) and
-        (updated.bbox should beTypedEqualTo(update.bbox))
+        (updated.bbox should beTypedEqualTo(update.bbox)) and
+        (updatedProperties should beTypedEqualTo(updateProperties))
   }
 
   def patchItemExpectation = prop { (stacCollection: StacCollection, stacItem: StacItem) =>

application/src/test/scala/com/azavea/franklin/api/services/FiltersFor.scala

@@ -1,22 +1,19 @@
 package com.azavea.franklin.api.services
 
-import cats.Semigroup
 import cats.data.NonEmptyList
 import cats.syntax.all._
+import cats.{Monoid, Semigroup}
 import com.azavea.franklin.database.SearchFilters
 import com.azavea.stac4s.jvmTypes.TemporalExtent
-import com.azavea.stac4s.{StacCollection, StacItem, TwoDimBbox}
+import com.azavea.stac4s.{ItemDatetime, StacCollection, StacItem, TwoDimBbox}
 import geotrellis.vector.Extent
-import io.circe.optics._
 import io.circe.syntax._
 
 import java.time.Instant
 
 object FiltersFor {
 
-  val datetimePrism = JsonPath.root.datetime.as[Instant]
-
-  implicit val searchFilterSemigroup: Semigroup[SearchFilters] = new Semigroup[SearchFilters] {
+  implicit val searchFilterMonoid: Monoid[SearchFilters] = new Monoid[SearchFilters] {
 
     def combine(x: SearchFilters, y: SearchFilters): SearchFilters = SearchFilters(
       x.bbox orElse y.bbox,
@@ -28,6 +25,17 @@ object FiltersFor {
       x.query |+| y.query,
       x.next orElse y.next
     )
+
+    def empty: SearchFilters = SearchFilters(
+      None,
+      None,
+      None,
+      List.empty,
+      List.empty,
+      None,
+      Map.empty,
+      None
+    )
   }
 
   def bboxFilterFor(item: StacItem): SearchFilters = {
@@ -51,21 +59,34 @@ object FiltersFor {
     )
   }
 
-  def timeFilterFor(item: StacItem): Option[SearchFilters] =
-    datetimePrism.getOption(item.properties.asJson) map { instant =>
-      TemporalExtent(instant.minusSeconds(60), Some(instant.plusSeconds(60)))
-    } map { temporalExtent =>
-      SearchFilters(
-        None,
-        Some(temporalExtent),
-        None,
-        Nil,
-        Nil,
-        None,
-        Map.empty,
-        None
-      )
+  def timeFilterFor(item: StacItem): SearchFilters = {
+    val temporalExtent = item.properties.datetime match {
+      case ItemDatetime.PointInTime(instant) =>
+        TemporalExtent(instant.minusSeconds(60), Some(instant.plusSeconds(60)))
+      case ItemDatetime.TimeRange(start, end) =>
+        val milli = start.toEpochMilli % 3
+        if (milli == 0) {
+          // test start and end with full overlap
+          TemporalExtent(start.minusSeconds(60), Some(end.plusSeconds(60)))
+        } else if (milli == 1) {
+          // test start before the range start with open end
+          TemporalExtent(start.minusSeconds(60), None)
+        } else {
+          // test end after the range end with open start
+          TemporalExtent(None, end.plusSeconds(60))
+        }
     }
+    SearchFilters(
+      None,
+      Some(temporalExtent),
+      None,
+      Nil,
+      Nil,
+      None,
+      Map.empty,
+      None
+    )
+  }
 
   def geomFilterFor(item: StacItem): SearchFilters = SearchFilters(
     None,
@@ -118,21 +139,34 @@ object FiltersFor {
     )
   }
 
-  def timeFilterExcluding(item: StacItem): Option[SearchFilters] =
-    datetimePrism.getOption(item.properties.asJson) map { instant =>
-      TemporalExtent(instant.minusSeconds(60), Some(instant.minusSeconds(30)))
-    } map { temporalExtent =>
-      SearchFilters(
-        None,
-        Some(temporalExtent),
-        None,
-        Nil,
-        Nil,
-        None,
-        Map.empty,
-        None
-      )
+  def timeFilterExcluding(item: StacItem): SearchFilters = {
+    val temporalExtent = item.properties.datetime match {
+      case ItemDatetime.PointInTime(instant) =>
+        TemporalExtent(instant.minusSeconds(60), Some(instant.minusSeconds(30)))
+      case ItemDatetime.TimeRange(start, end) =>
+        val milli = start.toEpochMilli % 3
+        if (milli == 0) {
+          // test no intersection with range
+          TemporalExtent(start.minusSeconds(60), Some(start.minusSeconds(30)))
+        } else if (milli == 1) {
+          // test start after the range end with open end
+          TemporalExtent(end.plusSeconds(60), None)
+        } else {
+          // test end before the range start with open start
+          TemporalExtent(None, start.minusSeconds(60))
+        }
     }
+    SearchFilters(
+      None,
+      Some(temporalExtent),
+      None,
+      Nil,
+      Nil,
+      None,
+      Map.empty,
+      None
+    )
+  }
 
   def geomFilterExcluding(item: StacItem): SearchFilters = {
     val itemGeomBbox = item.geometry.getEnvelopeInternal()
@@ -176,17 +210,14 @@ object FiltersFor {
   )
 
   def inclusiveFilters(collection: StacCollection, item: StacItem): SearchFilters = {
-    val filters: NonEmptyList[Option[SearchFilters]] = NonEmptyList
+    val filters: NonEmptyList[SearchFilters] = NonEmptyList
       .of(
-        bboxFilterFor(item).some,
+        bboxFilterFor(item),
         timeFilterFor(item),
-        geomFilterFor(item).some,
-        collectionFilterFor(collection).some,
-        itemFilterFor(item).some
+        geomFilterFor(item),
+        collectionFilterFor(collection),
+        itemFilterFor(item)
       )
-    val concatenated = filters.combineAll
-    // guaranteed to succeed, since most of the filters are being converted into options
-    // just to cooperate with timeFilterFor
-    concatenated.get
+    filters.combineAll
   }
 }