HDX-9881 Udpating ids in dataset and resource endpoints

hdx_hapi/config/doc_snippets.py

@@ -16,12 +16,12 @@
 DOC_GENDER_DESCRIPTION = 'Filter the response by the gender description.'
 DOC_POPULATION_GROUP = 'Filter the response by the population group. '
 DOC_POPULATION_STATUS = 'Filter the response by status. POP (population), AFF (affected), INN (in need), TGT (targeted), REA (reached) or all (all)'
-DOC_HDX_DATASET_ID = 'Filter the response by the dataset ID (hdx_id), which is a unique and fixed identifier of a Dataset on HDX. A URL in the pattern of `https://data.humdata.org/dataset/[hdx_id]` will load the dataset page on HDX.'
-DOC_HDX_DATASET_NAME = 'Filter the response by the URL-safe name (hdx_stub) of the dataset as displayed on HDX. This name is unique but can change. A URL in the pattern of `https://data.humdata.org/dataset/[hdx_stub]` will load the dataset page on HDX.'
+DOC_HDX_DATASET_ID = 'Filter the response by the dataset ID (dataset_hdx_id), which is a unique and fixed identifier of a Dataset on HDX. A URL in the pattern of `https://data.humdata.org/dataset/[dataset_hdx_id]` will load the dataset page on HDX.'
+DOC_HDX_DATASET_NAME = 'Filter the response by the URL-safe name (dataset_hdx_stub) of the dataset as displayed on HDX. This name is unique but can change. A URL in the pattern of `https://data.humdata.org/dataset/[dataset_hdx_stub]` will load the dataset page on HDX.'
 DOC_HDX_DATASET_TITLE = 'Filter the response by the title of the dataset as it appears in the HDX interface. This name is not unique and can change.'
 DOC_HDX_PROVIDER_STUB = "Filter the response by the code of the provider (organization) of the dataset on HDX. A URL in the pattern of `https://data.humdata.org/organization/[hdx_provider_stub]` will load the provider's page on HDX."
 DOC_HDX_PROVIDER_NAME = 'Filter the response by the display name of the provider (organization) of the dataset on HDX.'
-DOC_HDX_RESOURCE_ID = 'Filter the response by the resource ID (hdx_id), which is a unique and fixed identifier of a resource on HDX. A URL in the pattern of `https://data.humdata.org/dataset/[dataset_hdx_id]/resource/[hdx_id]` will load the resource page on HDX.'
+DOC_HDX_RESOURCE_ID = 'Filter the response by the resource ID (hdx_id), which is a unique and fixed identifier of a resource on HDX. A URL in the pattern of `https://data.humdata.org/dataset/[dataset_hdx_id]/resource/[resource_hdx_id]` will load the resource page on HDX.'
 DOC_HDX_RESOURCE_FORMAT = 'Filter the response by the format of the resource on HDX. These are typically file formats (i.e. CSV, XLSX), but can also include APIs and web apps.'
 DOC_HDX_RESOURCE_HXL = (
     'Filter the response by whether or not the resource contains <a href="https://hxlstandard.org/">HXL tags</a>.'

hdx_hapi/db/dao/dataset_view_dao.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Optional
 
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy import select
@@ -13,32 +14,33 @@
 async def datasets_view_list(
     pagination_parameters: PaginationParams,
     db: AsyncSession,
-    hdx_id: str = None,
-    hdx_stub: str = None,
-    title: str = None,
-    hdx_provider_stub: str = None,
-    hdx_provider_name: str = None,
+    dataset_hdx_id: Optional[str] = None,
+    dataset_hdx_stub: Optional[str] = None,
+    dataset_hdx_title: Optional[str] = None,
+    hdx_provider_stub: Optional[str] = None,
+    hdx_provider_name: Optional[str] = None,
 ):
     logger.info(
-        f'datasets_view_list called with params: hdx_id={hdx_id}, hdx_stub={hdx_stub}, title={title}, '
-        f'hdx_provider_stub={hdx_provider_stub}, hdx_provider_name={hdx_provider_name}'
+        f'datasets_view_list called with params: dataset_hdx_id={dataset_hdx_id}, dataset_hdx_stub={dataset_hdx_stub}, '
+        f'dataset_hdx_title={dataset_hdx_title}, hdx_provider_stub={hdx_provider_stub}, '
+        f'hdx_provider_name={hdx_provider_name}'
     )
 
     query = select(DatasetView)
-    if hdx_id:
-        query = query.where(DatasetView.hdx_id == hdx_id)
-    if hdx_stub:
-        query = query.where(DatasetView.hdx_stub == hdx_stub)
-    if title:
-        query = query.where(DatasetView.title.icontains(title))
+    if dataset_hdx_id:
+        query = query.where(DatasetView.dataset_hdx_id == dataset_hdx_id)
+    if dataset_hdx_stub:
+        query = query.where(DatasetView.dataset_hdx_stub == dataset_hdx_stub)
+    if dataset_hdx_title:
+        query = query.where(DatasetView.dataset_hdx_title.icontains(dataset_hdx_title))
     if hdx_provider_stub:
         query = case_insensitive_filter(query, DatasetView.hdx_provider_stub, hdx_provider_stub)
     if hdx_provider_name:
         query = query.where(DatasetView.hdx_provider_name.icontains(hdx_provider_name))
 
     query = apply_pagination(query, pagination_parameters)
 
-    logger.debug(f'Executing SQL query: {query}')
+    logger.info(f'Executing SQL query: {query}')
 
     result = await db.execute(query)
     datasets = result.scalars().all()

hdx_hapi/db/dao/resource_view_dao.py

@@ -10,22 +10,22 @@
 async def resources_view_list(
     pagination_parameters: PaginationParams,
     db: AsyncSession,
-    hdx_id: str = None,
+    resource_hdx_id: str = None,
     format: str = None,
     update_date_min: datetime = None,
     update_date_max: datetime = None,
     is_hxl: bool = None,
     hapi_updated_date_min: datetime = None,
     hapi_updated_date_max: datetime = None,
-    dataset_title: str = None,
+    dataset_hdx_title: str = None,
     dataset_hdx_id: str = None,
     dataset_hdx_stub: str = None,
     dataset_hdx_provider_stub: str = None,
     dataset_hdx_provider_name: str = None,
 ):
     query = select(ResourceView)
-    if hdx_id:
-        query = query.where(ResourceView.hdx_id == hdx_id)
+    if resource_hdx_id:
+        query = query.where(ResourceView.resource_hdx_id == resource_hdx_id)
     if format:
         query = query.where(ResourceView.format == format)
     if update_date_min:
@@ -38,8 +38,8 @@ async def resources_view_list(
         query = query.where(ResourceView.hapi_updated_date >= hapi_updated_date_min)
     if hapi_updated_date_max:
         query = query.where(ResourceView.hapi_updated_date < hapi_updated_date_max)
-    if dataset_title:
-        query = query.where(ResourceView.dataset_title == dataset_title)
+    if dataset_hdx_title:
+        query = query.where(ResourceView.dataset_hdx_title.icontains(dataset_hdx_title))
     if dataset_hdx_id:
         query = query.where(ResourceView.dataset_hdx_id == dataset_hdx_id)
     if dataset_hdx_stub:

hdx_hapi/db/models/views/all_views.py

@@ -131,9 +131,9 @@ class CurrencyView(Base):
 
 class DatasetView(Base):
     __table__ = dataset_view
-    hdx_id: Mapped[str] = column_property(dataset_view.c.hdx_id)
-    hdx_stub: Mapped[str] = column_property(dataset_view.c.hdx_stub)
-    title: Mapped[str] = column_property(dataset_view.c.title)
+    dataset_hdx_id: Mapped[str] = column_property(dataset_view.c.hdx_id)
+    dataset_hdx_stub: Mapped[str] = column_property(dataset_view.c.hdx_stub)
+    dataset_hdx_title: Mapped[str] = column_property(dataset_view.c.title)
     hdx_provider_stub: Mapped[str] = column_property(dataset_view.c.hdx_provider_stub)
     hdx_provider_name: Mapped[str] = column_property(dataset_view.c.hdx_provider_name)
 
@@ -363,7 +363,7 @@ class RefugeesView(Base):
 
 class ResourceView(Base):
     __table__ = resource_view
-    hdx_id: Mapped[str] = column_property(resource_view.c.hdx_id)
+    resource_hdx_id: Mapped[str] = column_property(resource_view.c.hdx_id)
     dataset_hdx_id: Mapped[str] = column_property(resource_view.c.dataset_hdx_id)
     name: Mapped[str] = column_property(resource_view.c.name)
     format: Mapped[str] = column_property(resource_view.c.format)
@@ -372,7 +372,7 @@ class ResourceView(Base):
     download_url: Mapped[str] = column_property(resource_view.c.download_url)
     hapi_updated_date: Mapped[DateTime] = column_property(resource_view.c.hapi_updated_date)
     dataset_hdx_stub: Mapped[str] = column_property(resource_view.c.dataset_hdx_stub)
-    dataset_title: Mapped[str] = column_property(resource_view.c.dataset_title)
+    dataset_hdx_title: Mapped[str] = column_property(resource_view.c.dataset_title)
     dataset_hdx_provider_stub: Mapped[str] = column_property(resource_view.c.dataset_hdx_provider_stub)
     dataset_hdx_provider_name: Mapped[str] = column_property(resource_view.c.dataset_hdx_provider_name)
 

hdx_hapi/endpoints/get_hdx_metadata.py

@@ -1,5 +1,5 @@
 from datetime import date
-from typing import Annotated
+from typing import Annotated, Optional
 from fastapi import Depends, Query, APIRouter
 from pydantic import NaiveDatetime
 
@@ -54,11 +54,11 @@
 async def get_datasets(
     common_parameters: Annotated[CommonEndpointParams, Depends(common_endpoint_parameters)],
     db: AsyncSession = Depends(get_db),
-    hdx_id: Annotated[str, Query(max_length=36, description=f'{DOC_HDX_DATASET_ID}')] = None,
-    hdx_stub: Annotated[str, Query(max_length=128, description=f'{DOC_HDX_DATASET_NAME}')] = None,
-    title: Annotated[str, Query(max_length=1024, description=f'{DOC_HDX_DATASET_TITLE}')] = None,
-    hdx_provider_stub: Annotated[str, Query(max_length=128, description=f'{DOC_HDX_PROVIDER_STUB}')] = None,
-    hdx_provider_name: Annotated[str, Query(max_length=512, description=f'{DOC_HDX_PROVIDER_NAME}')] = None,
+    dataset_hdx_id: Annotated[Optional[str], Query(max_length=36, description=f'{DOC_HDX_DATASET_ID}')] = None,
+    dataset_hdx_stub: Annotated[Optional[str], Query(max_length=128, description=f'{DOC_HDX_DATASET_NAME}')] = None,
+    dataset_hdx_title: Annotated[Optional[str], Query(max_length=1024, description=f'{DOC_HDX_DATASET_TITLE}')] = None,
+    hdx_provider_stub: Annotated[Optional[str], Query(max_length=128, description=f'{DOC_HDX_PROVIDER_STUB}')] = None,
+    hdx_provider_name: Annotated[Optional[str], Query(max_length=512, description=f'{DOC_HDX_PROVIDER_NAME}')] = None,
     output_format: OutputFormat = OutputFormat.JSON,
 ):
     """
@@ -68,9 +68,9 @@ async def get_datasets(
     result = await get_datasets_srv(
         pagination_parameters=common_parameters,
         db=db,
-        hdx_id=hdx_id,
-        hdx_stub=hdx_stub,
-        title=title,
+        dataset_hdx_id=dataset_hdx_id,
+        dataset_hdx_stub=dataset_hdx_stub,
+        dataset_hdx_title=dataset_hdx_title,
         hdx_provider_stub=hdx_provider_stub,
         hdx_provider_name=hdx_provider_name,
     )
@@ -91,7 +91,7 @@ async def get_datasets(
 async def get_resources(
     common_parameters: Annotated[CommonEndpointParams, Depends(common_endpoint_parameters)],
     db: AsyncSession = Depends(get_db),
-    hdx_id: Annotated[str, Query(max_length=36, description=f'{DOC_HDX_RESOURCE_ID}')] = None,
+    resource_hdx_id: Annotated[str, Query(max_length=36, description=f'{DOC_HDX_RESOURCE_ID}')] = None,
     format: Annotated[str, Query(max_length=32, description=f'{DOC_HDX_RESOURCE_FORMAT}')] = None,
     update_date_min: Annotated[
         NaiveDatetime | date,
@@ -108,7 +108,7 @@ async def get_resources(
     dataset_hdx_stub: Annotated[
         str, Query(max_length=128, description=f'{DOC_HDX_DATASET_IN_RESOURCE_NAME} {DOC_SEE_DATASET}')
     ] = None,
-    dataset_title: Annotated[
+    dataset_hdx_title: Annotated[
         str, Query(max_length=1024, description=f'{DOC_HDX_DATASET_TITLE} {DOC_SEE_DATASET}')
     ] = None,
     dataset_hdx_provider_stub: Annotated[
@@ -124,14 +124,14 @@ async def get_resources(
     result = await get_resources_srv(
         pagination_parameters=common_parameters,
         db=db,
-        hdx_id=hdx_id,
+        resource_hdx_id=resource_hdx_id,
         format=format,
         update_date_min=update_date_min,
         update_date_max=update_date_max,
         is_hxl=is_hxl,
         dataset_hdx_id=dataset_hdx_id,
         dataset_hdx_stub=dataset_hdx_stub,
-        dataset_title=dataset_title,
+        dataset_hdx_title=dataset_hdx_title,
         dataset_hdx_provider_stub=dataset_hdx_provider_stub,
         dataset_hdx_provider_name=dataset_hdx_provider_name,
     )

hdx_hapi/endpoints/models/hdx_metadata.py

@@ -13,9 +13,9 @@
 
 
 class DatasetResponse(HapiBaseModel):
-    hdx_id: str = Field(max_length=36)
-    hdx_stub: str = Field(max_length=128)
-    title: str = Field(max_length=1024)
+    dataset_hdx_id: str = Field(max_length=36)
+    dataset_hdx_stub: str = Field(max_length=128)
+    dataset_hdx_title: str = Field(max_length=1024)
     hdx_provider_stub: str = Field(max_length=128)
     hdx_provider_name: str = Field(max_length=512)
 
@@ -24,12 +24,12 @@ class DatasetResponse(HapiBaseModel):
     @computed_field
     @property
     def hdx_link(self) -> HttpUrl:
-        return get_dataset_url(dataset_id=self.hdx_id)
+        return get_dataset_url(dataset_id=self.dataset_hdx_id)
 
     @computed_field
     @property
     def hdx_api_link(self) -> HttpUrl:
-        return get_dataset_api_url(dataset_id=self.hdx_id)
+        return get_dataset_api_url(dataset_id=self.dataset_hdx_id)
 
     @computed_field
     @property
@@ -53,7 +53,7 @@ def list_of_fields(self) -> List[str]:
 
 class ResourceResponse(HapiBaseModel):
     # id: int
-    hdx_id: str = Field(max_length=36)
+    resource_hdx_id: str = Field(max_length=36)
     dataset_hdx_id: str = Field(max_length=36)
     name: str = Field(max_length=256)
     format: str = Field(max_length=32)
@@ -64,7 +64,7 @@ class ResourceResponse(HapiBaseModel):
 
     dataset_hdx_stub: str = Field(max_length=128)
 
-    dataset_title: str = Field(max_length=1024)
+    dataset_hdx_title: str = Field(max_length=1024)
     dataset_hdx_provider_stub: str = Field(max_length=128)
     dataset_hdx_provider_name: str = Field(max_length=512)
 
@@ -73,12 +73,12 @@ class ResourceResponse(HapiBaseModel):
     @computed_field
     @property
     def hdx_link(self) -> HttpUrl:
-        return get_resource_url(dataset_id=self.dataset_hdx_id, resource_id=self.hdx_id)
+        return get_resource_url(dataset_id=self.dataset_hdx_id, resource_id=self.resource_hdx_id)
 
     @computed_field
     @property
     def hdx_api_link(self) -> HttpUrl:
-        return get_resource_api_url(resource_id=self.hdx_id)
+        return get_resource_api_url(resource_id=self.resource_hdx_id)
 
     @computed_field
     @property

hdx_hapi/services/dataset_logic.py

@@ -1,3 +1,4 @@
+from typing import Optional
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from hdx_hapi.db.dao.dataset_view_dao import datasets_view_list
@@ -7,13 +8,18 @@
 async def get_datasets_srv(
     pagination_parameters: PaginationParams,
     db: AsyncSession,
-    hdx_id: str,
-    hdx_stub: str,
-    title: str = None,
-    hdx_provider_stub: str = None,
-    hdx_provider_name: str = None,
+    dataset_hdx_id: Optional[str] = None,
+    dataset_hdx_stub: Optional[str] = None,
+    dataset_hdx_title: Optional[str] = None,
+    hdx_provider_stub: Optional[str] = None,
+    hdx_provider_name: Optional[str] = None,
 ):
     return await datasets_view_list(
-        pagination_parameters=pagination_parameters, db=db, hdx_id=hdx_id, hdx_stub=hdx_stub, title=title,
-                hdx_provider_stub=hdx_provider_stub, hdx_provider_name=hdx_provider_name
+        pagination_parameters=pagination_parameters,
+        db=db,
+        dataset_hdx_id=dataset_hdx_id,
+        dataset_hdx_stub=dataset_hdx_stub,
+        dataset_hdx_title=dataset_hdx_title,
+        hdx_provider_stub=hdx_provider_stub,
+        hdx_provider_name=hdx_provider_name,
     )

hdx_hapi/services/resource_logic.py

@@ -8,14 +8,14 @@
 async def get_resources_srv(
     pagination_parameters: PaginationParams,
     db: AsyncSession,
-    hdx_id: str = None,
+    resource_hdx_id: str = None,
     format: str = None,
     update_date_min: datetime = None,
     update_date_max: datetime = None,
     is_hxl: bool = None,
     hapi_updated_date_min: datetime = None,
     hapi_updated_date_max: datetime = None,
-    dataset_title: str = None,
+    dataset_hdx_title: str = None,
     dataset_hdx_id: str = None,
     dataset_hdx_stub: str = None,
     dataset_hdx_provider_stub: str = None,
@@ -24,7 +24,7 @@ async def get_resources_srv(
     return await resources_view_list(
         pagination_parameters=pagination_parameters,
         db=db,
-        hdx_id=hdx_id,
+        resource_hdx_id=resource_hdx_id,
         format=format,
         update_date_min=update_date_min,
         update_date_max=update_date_max,
@@ -33,7 +33,7 @@ async def get_resources_srv(
         hapi_updated_date_max=hapi_updated_date_max,
         dataset_hdx_id=dataset_hdx_id,
         dataset_hdx_stub=dataset_hdx_stub,
-        dataset_title=dataset_title,
+        dataset_hdx_title=dataset_hdx_title,
         dataset_hdx_provider_stub=dataset_hdx_provider_stub,
         dataset_hdx_provider_name=dataset_hdx_provider_name,
     )

tests/test_endpoints/endpoint_data.py

@@ -59,15 +59,15 @@
     },
     '/api/v1/metadata/dataset': {
         'query_parameters': {
-            'hdx_id': '90deb235-1bf5-4bae-b231-3393222c2d01',
-            'title': 'Dataset #1',
+            'dataset_hdx_id': '90deb235-1bf5-4bae-b231-3393222c2d01',
+            'dataset_hdx_title': 'Dataset #1',
             'hdx_provider_stub': 'pRoViDeR01',
             'hdx_provider_name': 'Provider #1',
         },
         'expected_fields': [
-            'hdx_id',
-            'hdx_stub',
-            'title',
+            'dataset_hdx_id',
+            'dataset_hdx_stub',
+            'dataset_hdx_title',
             'hdx_provider_stub',
             'hdx_provider_name',
             'hdx_link',  # computed field
@@ -468,20 +468,20 @@
     },
     '/api/v1/metadata/resource': {
         'query_parameters': {
-            'hdx_id': '17acb541-9431-409a-80a8-50eda7e8ebab',
+            'resource_hdx_id': '17acb541-9431-409a-80a8-50eda7e8ebab',
             'name': 'resource-01.csv',
             'format': 'csv',
             'update_date_min': date(2023, 6, 1),
             'update_date_max': date(2023, 6, 3),
             'is_hxl': True,
             'hapi_updated_date': date(2023, 6, 2),
             'dataset_hdx_stub': 'dataset01',
-            'dataset_title': 'Dataset #1',
+            'dataset_hdx_title': 'Dataset #1',
             'dataset_hdx_provider_stub': 'pRoViDeR01',
             'dataset_hdx_provider_name': 'Provider #1',
         },
         'expected_fields': [
-            'hdx_id',
+            'resource_hdx_id',
             'dataset_hdx_id',
             'name',
             'format',
@@ -490,7 +490,7 @@
             'download_url',
             'hapi_updated_date',
             'dataset_hdx_stub',
-            'dataset_title',
+            'dataset_hdx_title',
             'dataset_hdx_provider_stub',
             'dataset_hdx_provider_name',
             'hdx_link',  # computed field