Feature/hdx 9818 prepare for new schema

hdx_hapi/config/config.py

@@ -1,8 +1,9 @@
-from dataclasses import dataclass
-from mixpanel import Mixpanel
 import logging
 import os
 
+from dataclasses import dataclass
+from mixpanel import Mixpanel
+
 from hdx_hapi.config.helper import create_pg_uri_from_env_without_protocol
 
 logger = logging.getLogger(__name__)
@@ -18,6 +19,7 @@ class Config:
     HDX_DATASET_URL: str
     HDX_DATASET_API_URL: str
     HDX_ORGANIZATION_URL: str
+    HDX_ORGANIZATION_API_URL: str
 
     HDX_RESOURCE_URL: str
     HDX_RESOURCE_API_URL: str
@@ -28,7 +30,7 @@ class Config:
 
     HAPI_IDENTIFIER_FILTERING: bool
 
-    HDX_MIXPANEL_TOKEN: str
+    MIXPANEL: Mixpanel
 
 
 CONFIG = None
@@ -41,6 +43,7 @@ def get_config() -> Config:
 
         sql_alchemy_asyncypg_db_uri = f'postgresql+asyncpg://{db_uri_without_protocol}'
         sql_alchemy_psycopg2_db_uri = f'postgresql+psycopg2://{db_uri_without_protocol}'
+        mixpanel_token = os.getenv('HDX_MIXPANEL_TOKEN', '')
         CONFIG = Config(
             SQL_ALCHEMY_ASYNCPG_DB_URI=sql_alchemy_asyncypg_db_uri,
             SQL_ALCHEMY_PSYCOPG2_DB_URI=sql_alchemy_psycopg2_db_uri,
@@ -52,14 +55,15 @@ def get_config() -> Config:
                 'HDX_RESOURCE_API_URL', '{domain}/api/action/resource_show?id={resource_id}'
             ),
             HDX_ORGANIZATION_URL=os.getenv('HDX_ORGANIZATION_URL', '{domain}/organization/{org_id}'),
+            HDX_ORGANIZATION_API_URL=os.getenv(
+                'HDX_ORGANIZATION_API_URL', '{domain}/api/action/organization_show?id={org_id}'
+            ),
             HAPI_READTHEDOCS_OVERVIEW_URL=os.getenv(
                 'HAPI_READTHEDOCS_OVERVIEW_URL', 'https://hdx-hapi.readthedocs.io/en/latest/'
             ),
             HAPI_SERVER_URL=os.getenv('HAPI_SERVER_URL', None),
             HAPI_IDENTIFIER_FILTERING=os.getenv('HAPI_IDENTIFIER_FILTERING', 'True').lower() == 'true',
-            HDX_MIXPANEL_TOKEN=os.getenv('HDX_MIXPANEL_TOKEN', ''),
+            MIXPANEL=Mixpanel(mixpanel_token) if mixpanel_token else None,
         )
 
     return CONFIG
-
-mixpanel = Mixpanel(get_config().HDX_MIXPANEL_TOKEN)

hdx_hapi/config/doc_snippets.py

@@ -7,9 +7,10 @@
 DOC_ADMIN2_CODE = 'Filter the response by the 2nd subnational administrative divisions. The admin2 codes refer to the p-codes in the <a href="https://data.humdata.org/dashboards/cod?">Common Operational Datasets</a>.'
 DOC_ADMIN2_NAME = 'Filter the response by the 1st subnational administrative divisions. The admin2 names refer to the <a href="https://data.humdata.org/dashboards/cod?">Common Operational Datasets</a>.'
 DOC_AGE_RANGE_SUMMARY = 'Get the list of age ranges used for disaggregating population data'
-DOC_AGE_RANGE_CODE = 'Filter the response by the age range. These are expressed as [start year]-[end year]. The end year is assumed to be inclusive, though that is not always explicit in the source data.'
+DOC_AGE_RANGE = 'Filter the response by the age range. These are expressed as [start year]-[end year]. The end year is assumed to be inclusive, though that is not always explicit in the source data.'
 DOC_GENDER_SUMMARY = 'Get the list of gender codes used for disaggregating population data'
 DOC_GENDER_CODE = 'Filter the response by the gender code.'
+DOC_GENDER = 'Filter the response by the gender.'
 DOC_GENDER_DESCRIPTION = 'Filter the response by the gender description.'
 DOC_HDX_DATASET_ID = 'Filter the response by the dataset ID, which is a unique and fixed identifier of a Dataset on HDX. A URL in the pattern of `https://data.humdata.org/dataset/[dataset id]` will load the dataset page on HDX.'
 DOC_HDX_DATASET_NAME = 'Filter the response by the URL-safe name of the dataset as displayed on HDX. This name is unique but can change. A URL in the pattern of `https://data.humdata.org/dataset/[dataset name]` will load the dataset page on HDX.'

hdx_hapi/db/dao/admin1_view_dao.py

@@ -1,26 +1,22 @@
-from datetime import datetime
 import logging
 
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy import select
 
-from hdx_hapi.db.models.views.db_admin1_view import Admin1View
-from hdx_hapi.db.dao.util.util import apply_pagination, case_insensitive_filter
-from hdx_hapi.endpoints.util.util import PaginationParams
+from hdx_hapi.db.models.views.all_views import Admin1View
+from hdx_hapi.db.dao.util.util import apply_pagination, apply_reference_period_filter, case_insensitive_filter
+from hdx_hapi.endpoints.util.util import PaginationParams, ReferencePeriodParameters
 
 
 logger = logging.getLogger(__name__)
 
 
 async def admin1_view_list(
     pagination_parameters: PaginationParams,
+    ref_period_parameters: ReferencePeriodParameters,
     db: AsyncSession,
     code: str = None,
     name: str = None,
-    hapi_updated_date_min: datetime = None,
-    hapi_updated_date_max: datetime = None,
-    hapi_replaced_date_min: datetime = None,
-    hapi_replaced_date_max: datetime = None,
     location_code: str = None,
     location_name: str = None,
 ):
@@ -37,19 +33,13 @@ async def admin1_view_list(
         query = case_insensitive_filter(query, Admin1View.code, code)
     if name:
         query = query.where(Admin1View.name.icontains(name))
-    if hapi_updated_date_min:
-        query = query.where(Admin1View.hapi_updated_date >= hapi_updated_date_min)
-    if hapi_updated_date_max:
-        query = query.where(Admin1View.hapi_updated_date < hapi_updated_date_max)
-    if hapi_replaced_date_min:
-        query = query.where(Admin1View.hapi_replaced_date >= hapi_replaced_date_min)
-    if hapi_replaced_date_max:
-        query = query.where(Admin1View.hapi_replaced_date < hapi_replaced_date_max)
     if location_code:
         query = case_insensitive_filter(query, Admin1View.location_code, location_code)
     if location_name:
         query = query.where(Admin1View.location_name.icontains(location_name))
 
+    query = apply_reference_period_filter(query, ref_period_parameters, Admin1View)
+
     query = apply_pagination(query, pagination_parameters)
 
     logger.debug(f'Executing SQL query: {query}')

hdx_hapi/db/dao/admin2_view_dao.py

@@ -1,26 +1,22 @@
-from datetime import datetime
 import logging
 
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy import select
 
-from hdx_hapi.db.models.views.db_admin2_view import Admin2View
-from hdx_hapi.db.dao.util.util import apply_pagination, case_insensitive_filter
-from hdx_hapi.endpoints.util.util import PaginationParams
+from hdx_hapi.db.models.views.all_views import Admin2View
+from hdx_hapi.db.dao.util.util import apply_pagination, apply_reference_period_filter, case_insensitive_filter
+from hdx_hapi.endpoints.util.util import PaginationParams, ReferencePeriodParameters
 
 
 logger = logging.getLogger(__name__)
 
 
 async def admin2_view_list(
     pagination_parameters: PaginationParams,
+    ref_period_parameters: ReferencePeriodParameters,
     db: AsyncSession,
     code: str = None,
     name: str = None,
-    hapi_updated_date_min: datetime = None,
-    hapi_updated_date_max: datetime = None,
-    hapi_replaced_date_min: datetime = None,
-    hapi_replaced_date_max: datetime = None,
     admin1_code: str = None,
     admin1_name: str = None,
     location_code: str = None,
@@ -39,14 +35,6 @@ async def admin2_view_list(
         query = case_insensitive_filter(query, Admin2View.code, code)
     if name:
         query = query.where(Admin2View.name.icontains(name))
-    if hapi_updated_date_min:
-        query = query.where(Admin2View.hapi_updated_date >= hapi_updated_date_min)
-    if hapi_updated_date_max:
-        query = query.where(Admin2View.hapi_updated_date < hapi_updated_date_max)
-    if hapi_replaced_date_min:
-        query = query.where(Admin2View.hapi_replaced_date >= hapi_replaced_date_min)
-    if hapi_replaced_date_max:
-        query = query.where(Admin2View.hapi_replaced_date < hapi_replaced_date_max)
     if admin1_code:
         query = case_insensitive_filter(query, Admin2View.admin1_code, admin1_code)
     if admin1_name:
@@ -56,6 +44,8 @@ async def admin2_view_list(
     if location_name:
         query = query.where(Admin2View.location_name.icontains(location_name))
 
+    query = apply_reference_period_filter(query, ref_period_parameters, Admin2View)
+
     query = apply_pagination(query, pagination_parameters)
 
     logger.debug(f'Executing SQL query: {query}')

hdx_hapi/db/dao/conflict_event_view_dao.py

@@ -0,0 +1,68 @@
+import logging
+from typing import Optional, Sequence
+
+from hapi_schema.utils.enums import EventType
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy import select
+
+from hdx_hapi.db.models.views.all_views import ConflictEventView
+from hdx_hapi.db.dao.util.util import (
+    apply_location_admin_filter,
+    apply_pagination,
+    apply_reference_period_filter,
+)
+from hdx_hapi.endpoints.util.util import PaginationParams, ReferencePeriodParameters
+
+
+logger = logging.getLogger(__name__)
+
+
+async def conflict_event_view_list(
+    pagination_parameters: PaginationParams,
+    ref_period_parameters: ReferencePeriodParameters,
+    db: AsyncSession,
+    event_type: Optional[EventType] = None,
+    location_ref: Optional[int] = None,
+    location_code: Optional[str] = None,
+    location_name: Optional[str] = None,
+    admin1_ref: Optional[int] = None,
+    admin1_code: Optional[str] = None,
+    admin1_name: Optional[str] = None,
+    admin1_is_unspecified: Optional[bool] = None,
+    admin2_ref: Optional[int] = None,
+    admin2_code: Optional[str] = None,
+    admin2_name: Optional[str] = None,
+    admin2_is_unspecified: Optional[bool] = None,
+) -> Sequence[ConflictEventView]:
+    query = select(ConflictEventView)
+    if event_type:
+        query = query.where(ConflictEventView.event_type == event_type)
+
+    query = apply_location_admin_filter(
+        query,
+        ConflictEventView,
+        location_ref,
+        location_code,
+        location_name,
+        admin1_ref,
+        admin1_code,
+        admin1_name,
+        admin1_is_unspecified,
+        admin2_ref,
+        admin2_code,
+        admin2_name,
+        admin2_is_unspecified,
+    )
+
+    query = apply_reference_period_filter(query, ref_period_parameters, ConflictEventView)
+
+    query = apply_pagination(query, pagination_parameters)
+
+    logger.debug(f'Executing SQL query: {query}')
+
+    result = await db.execute(query)
+    conflict_events = result.scalars().all()
+
+    logger.info(f'Retrieved {len(conflict_events)} rows from the database')
+
+    return conflict_events

hdx_hapi/db/dao/dataset_view_dao.py

@@ -3,7 +3,7 @@
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy import select
 
-from hdx_hapi.db.models.views.db_dataset_view import DatasetView
+from hdx_hapi.db.models.views.all_views import DatasetView
 from hdx_hapi.db.dao.util.util import apply_pagination, case_insensitive_filter
 from hdx_hapi.endpoints.util.util import PaginationParams
 

hdx_hapi/db/dao/food_security_view_dao.py

@@ -3,7 +3,7 @@
 from sqlalchemy import select
 
 from hdx_hapi.db.models.views.db_food_security_view import FoodSecurityView
-from hdx_hapi.db.dao.util.util import apply_pagination, case_insensitive_filter
+from hdx_hapi.db.dao.util.util import apply_location_admin_filter, apply_pagination, case_insensitive_filter
 from hdx_hapi.endpoints.util.util import PaginationParams
 
 
@@ -29,6 +29,7 @@ async def food_security_view_list(
     admin2_name: str = None,
     admin2_is_unspecified: bool = None,
     admin1_ref: int = None,
+    admin2_ref: int = None,
 ):
     query = select(FoodSecurityView)
 
@@ -50,26 +51,22 @@ async def food_security_view_list(
         query = query.where(FoodSecurityView.hapi_replaced_date >= hapi_replaced_date_min)
     if hapi_replaced_date_max:
         query = query.where(FoodSecurityView.hapi_replaced_date < hapi_replaced_date_max)
-    if location_code:
-        query = case_insensitive_filter(query, FoodSecurityView.location_code, location_code)
-    if location_name:
-        query = query.where(FoodSecurityView.location_name.icontains(location_name))
-    if admin1_name:
-        query = query.where(FoodSecurityView.admin1_name.icontains(admin1_name))
-    if admin1_code:
-        query = case_insensitive_filter(query, FoodSecurityView.admin1_code, admin1_code)
-    if admin1_is_unspecified is not None:
-        query = query.where(FoodSecurityView.admin1_is_unspecified == admin1_is_unspecified)
-    if location_ref:
-        query = query.where(FoodSecurityView.location_ref == location_ref)
-    if admin2_code:
-        query = case_insensitive_filter(query, FoodSecurityView.admin2_code, admin2_code)
-    if admin2_name:
-        query = query.where(FoodSecurityView.admin2_name.icontains(admin2_name))
-    if admin2_is_unspecified is not None:
-        query = query.where(FoodSecurityView.admin2_is_unspecified == admin2_is_unspecified)
-    if admin1_ref:
-        query = query.where(FoodSecurityView.admin1_ref == admin1_ref)
+
+    query = apply_location_admin_filter(
+        query,
+        FoodSecurityView,
+        location_ref,
+        location_code,
+        location_name,
+        admin1_ref,
+        admin1_code,
+        admin1_name,
+        admin1_is_unspecified,
+        admin2_ref,
+        admin2_code,
+        admin2_name,
+        admin2_is_unspecified,
+    )
 
     query = apply_pagination(query, pagination_parameters)