Improve temporal and spatial support

app/domain/entities.py

@@ -1,5 +1,6 @@
 import dataclasses
 from datetime import datetime
+from dateutil.parser import isoparse
 
 
 @dataclasses.dataclass
@@ -22,15 +23,15 @@ class Organization(EntityBase):
     description: str
     url: str
     orga_sp: int
-    created_at: str
+    created_at: datetime.date
     followers: int
     datasets: int
 
     acronym: str = None
 
     def __post_init__(self):
-        if isinstance(self.created_at, datetime):
-            self.created_at = self.created_at.strftime('%Y-%m-%d')
+        if isinstance(self.created_at, str):
+            self.created_at = isoparse(self.created_at)
 
 
 @dataclasses.dataclass
@@ -39,7 +40,7 @@ class Dataset(EntityBase):
     title: str
     acronym: str
     url: str
-    created_at: str
+    created_at: datetime.date
     views: int
     followers: int
     reuses: int
@@ -48,27 +49,31 @@ class Dataset(EntityBase):
     concat_title_org: str
     description: str
 
-    temporal_coverage_start: str = None
-    temporal_coverage_end: str = None
+    temporal_coverage_start: datetime.date = None
+    temporal_coverage_end: datetime.date = None
     granularity: str = None
-    geozones: str = None
+    geozone: str = None
 
     orga_sp: int = None
     orga_followers: int = None
     organization_id: str = None
     organization: str = None
 
     def __post_init__(self):
-        if isinstance(self.created_at, datetime):
-            self.created_at = self.created_at.strftime('%Y-%m-%d')
+        if isinstance(self.created_at, str):
+            self.created_at = isoparse(self.created_at)
+        if isinstance(self.temporal_coverage_start, str):
+            self.temporal_coverage_start = isoparse(self.temporal_coverage_start)
+        if isinstance(self.temporal_coverage_end, str):
+            self.temporal_coverage_end = isoparse(self.temporal_coverage_end)
 
 
 @dataclasses.dataclass
 class Reuse(EntityBase):
     id: str
     title: str
     url: str
-    created_at: str
+    created_at: datetime.date
     views: int
     followers: int
     datasets: int
@@ -80,5 +85,5 @@ class Reuse(EntityBase):
     organization: str = None
 
     def __post_init__(self):
-        if isinstance(self.created_at, datetime):
-            self.created_at = self.created_at.strftime('%Y-%m-%d')
+        if isinstance(self.created_at, str):
+            self.created_at = isoparse(self.created_at)

app/infrastructure/kafka_consumer.py

@@ -57,7 +57,7 @@ def load_from_dict(cls, data):
         data["organization"] = data["organization"].get('name') if data["organization"] else None
 
         data["concat_title_org"] = data["title"] + (' ' + data["organization"] if data["organization"] else '')
-        data["geozones"] = ''  # TODO
+        data["geozone"] = [zone.get("id") for zone in data.get("geozones", [])]
         return super().load_from_dict(data)
 
 
@@ -83,7 +83,7 @@ def consume_messages(consumer, es):
         key = message.key
         index = message.topic
 
-        logging.warning(f'Message recieved with key: {key} and value: {value}')
+        logging.info(f'Message recieved with key: {key} and value: {value}')
 
         if val_utf8 != 'null':
             if index == 'dataset':

app/infrastructure/search_clients.py

@@ -1,6 +1,6 @@
 from typing import Tuple, Optional, List
 from elasticsearch.exceptions import NotFoundError
-from elasticsearch_dsl import Index, Document, Integer, Text, tokenizer, token_filter, analyzer, query
+from elasticsearch_dsl import Index, Date, Document, Integer, Keyword, Text, tokenizer, token_filter, analyzer, query
 from elasticsearch_dsl.connections import connections
 from app.domain.entities import Dataset, Organization, Reuse
 from app.config import Config
@@ -28,7 +28,7 @@ class SearchableOrganization(Document):
     description = Text(analyzer=dgv_analyzer)
     url = Text()
     orga_sp = Integer()
-    created_at = Text()
+    created_at = Date()
     followers = Integer()
     datasets = Integer()
 
@@ -39,7 +39,7 @@ class Index:
 class SearchableReuse(Document):
     title = Text(analyzer=dgv_analyzer)
     url = Text()
-    created_at = Text()
+    created_at = Date()
     orga_followers = Integer()
     views = Integer()
     followers = Integer()
@@ -57,7 +57,7 @@ class SearchableDataset(Document):
     title = Text(analyzer=dgv_analyzer)
     acronym = Text()
     url = Text()
-    created_at = Text()
+    created_at = Date()
     orga_sp = Integer()
     orga_followers = Integer()
     views = Integer()
@@ -67,10 +67,10 @@ class SearchableDataset(Document):
     resources_count = Integer()
     concat_title_org = Text(analyzer=dgv_analyzer)
     organization_id = Text()
-    temporal_coverage_start = Text()
-    temporal_coverage_end = Text()
+    temporal_coverage_start = Date()
+    temporal_coverage_end = Date()
     granularity = Text()
-    geozones = Text()
+    geozone = Keyword(multi=True)
     description = Text(analyzer=dgv_analyzer)
     organization = Text(analyzer=dgv_analyzer)
 

tests/test_search_client.py

@@ -1,4 +1,6 @@
+import datetime
 import time
+
 from app.domain.entities import Dataset, Organization, Reuse
 
 
@@ -25,7 +27,7 @@ def test_general_search_with_and_without_query(app, client, search_client, faker
             temporal_coverage_start=None,
             temporal_coverage_end=None,
             granularity=None,
-            geozones=None,
+            geozone=None,
             description=faker.sentence(nb_words=10),
             organization=organization
         ))
@@ -98,7 +100,7 @@ def test_search_with_filters(app, client, search_client, faker):
             temporal_coverage_start=None,
             temporal_coverage_end=None,
             granularity=None,
-            geozones=None,
+            geozone=None,
             description=faker.sentence(nb_words=10),
             organization=organization
         ))
@@ -147,10 +149,10 @@ def test_search_datasets_with_temporal_filters(app, client, search_client, faker
             resources_count=faker.random_int(min=1, max=15),
             concat_title_org=title + ' ' + acronym + ' ' + organization,
             organization_id=faker.md5(),
-            temporal_coverage_start='2021-12-02' if i % 2 else '2020-02-24',
-            temporal_coverage_end='2022-01-13' if i % 2 else '2020-03-13',
+            temporal_coverage_start=datetime.date(2021, 12, 2) if i % 2 else datetime.date(2020, 2, 24),
+            temporal_coverage_end=datetime.date(2022, 1, 13) if i % 2 else datetime.date(2020, 3, 13),
             granularity=None,
-            geozones=None,
+            geozone=None,
             description=faker.sentence(nb_words=10),
             organization=organization
         ))
@@ -160,7 +162,17 @@ def test_search_datasets_with_temporal_filters(app, client, search_client, faker
     results_number, res = search_client.query_datasets('test', 0, 20, {})
     assert results_number == 4
     results_number, res = search_client.query_datasets('test', 0, 20, {
-        'temporal_coverage_start': '2020-01-29',
-        'temporal_coverage_end': '2020-04-15'
+        'temporal_coverage_start': datetime.date(2020, 2, 25),
+        'temporal_coverage_end': datetime.date(2020, 3, 10)
+    })
+    assert results_number == 2
+    results_number, res = search_client.query_datasets('test', 0, 20, {
+        'temporal_coverage_start': datetime.date(2022, 1, 1),
+        'temporal_coverage_end': datetime.date(2022, 1, 10)
     })
     assert results_number == 2
+    results_number, res = search_client.query_datasets('test', 0, 20, {
+        'temporal_coverage_start': datetime.date(1970, 1, 1),
+        'temporal_coverage_end': datetime.date(2022, 12, 31)
+    })
+    assert results_number == 0