Merge pull request #305 from OCHA-DAP/dev

dev into prod

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "src/hapi-schema"]
-path = src/hapi-schema
-url = https://github.com/OCHA-DAP/hapi-sqlalchemy-schema.git
+[submodule "src/hapi-schema"]
+path = src/hapi-schema
+url = https://github.com/OCHA-DAP/hapi-sqlalchemy-schema.git

.vscode/launch.json

@@ -12,6 +12,24 @@
             "console": "integratedTerminal",
             "justMyCode": true,
             "env":{"HAPI_USE_VAT": "false"}
+        },
+        {
+            "name": "generate-views",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "/srv/hapi/src/hapi-schema/src/hapi_schema/utils/hapi_views_code_generator.py",
+            "console": "integratedTerminal",
+            "justMyCode": true,
+            "env":{"HAPI_USE_VAT": "false"}
+        },
+        {
+            "name": "generate-vat",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "/srv/hapi/src/hapi-schema/src/hapi_schema/utils/view_as_table_code_generator.py",
+            "console": "integratedTerminal",
+            "justMyCode": true,
+            "env":{"HAPI_USE_VAT": "true"}
         }
     ],
 

docs/index.md

@@ -18,6 +18,21 @@ details the data coverage that we have achieved at present, and to which
 administrative level the data is available: national (admin 0), admin 1,
 or admin 2.
 
+## Accessing the Data
+
+You can find out more about HDX HAPI in the 
+[Getting Started](getting-started.md) and 
+[API documentation](https://hapi.humdata.org) pages. 
+
+There are also
+[HDX HAPI datasets on HDX](https://data.humdata.org/organization/hdx-hapi)
+containing the data available from the API. Some datasets have warning and 
+error columns. Warnings typically indicate corrections have been made to 
+the data or show things to look out for. Rows with only warnings are considered 
+complete, and are available from the API. Errors usually mean that the data 
+is incomplete or unusable. Rows with any errors are not present in the API but 
+have been included for transparency.
+
 ## Latest Changes
 
 _Note: this is only a 10-line snippet, for full details please refer

hdx_hapi/db/dao/availability_view_dao.py

@@ -6,26 +6,21 @@
 from sqlalchemy.orm import Mapped
 
 from hdx_hapi.db.models.views.vat_or_view import AvailabilityView
-from hdx_hapi.db.dao.util.util import apply_pagination, case_insensitive_filter
-from hdx_hapi.endpoints.util.util import AdminLevel, PaginationParams
+from hdx_hapi.db.dao.util.util import apply_location_admin_filter, apply_pagination, case_insensitive_filter
+from hdx_hapi.endpoints.util.util import CommonLocationParameters, PaginationParams
 
 logger = logging.getLogger(__name__)
 _UNSPECIFIED = 'UNSPECIFIED'
 
+
 async def availability_view_list(
     pagination_parameters: PaginationParams,
+    common_location_params: CommonLocationParameters,
     db: AsyncSession,
     category: Optional[str] = None,
     subcategory: Optional[str] = None,
-    location_name: Optional[str] = None,
-    location_code: Optional[str] = None,
-    admin1_name: Optional[str] = None,
-    admin1_code: Optional[str] = None,
-    admin2_name: Optional[str] = None,
-    admin2_code: Optional[str] = None,
     hapi_updated_date_min: Optional[datetime.datetime | datetime.date] = None,
     hapi_updated_date_max: Optional[datetime.datetime | datetime.date] = None,
-    admin_level: Optional[AdminLevel] = None,
 ):
     logger.info(f'availability_view_list called with params: {locals()}')
 
@@ -35,34 +30,28 @@ async def availability_view_list(
     if subcategory:
         query = case_insensitive_filter(query, AvailabilityView.subcategory, subcategory)
 
-    if location_code:
-        query = case_insensitive_filter(query, AvailabilityView.location_code, location_code)
-    if location_name:
-        query = query.where(AvailabilityView.location_name.icontains(location_name))
-    if admin1_code:
-        query = case_insensitive_filter(query, AvailabilityView.admin1_code, admin1_code)
-    if admin1_name:
-        query = query.where(AvailabilityView.admin1_name.icontains(admin1_name))
-    if admin2_code:
-        query = case_insensitive_filter(query, AvailabilityView.admin2_code, admin2_code)
-    if admin2_name:
-        query = query.where(AvailabilityView.admin2_name.icontains(admin2_name))
+    query = apply_location_admin_filter(
+        query,
+        AvailabilityView,
+        common_location_params,
+    )
+
     if hapi_updated_date_min:
         query = query.where(AvailabilityView.hapi_updated_date >= hapi_updated_date_min)
     if hapi_updated_date_max:
         query = query.where(AvailabilityView.hapi_updated_date < hapi_updated_date_max)
 
     # Admin level filtering. Filtering by admin level is handled differently for the data availability table
     # beause we don't have the adminX_is_unspecified fields.
-    if admin_level == AdminLevel.ZERO:
-        query = filter_is_unspecified(query, AvailabilityView.admin1_name)
-        query = filter_is_unspecified(query, AvailabilityView.admin2_name)
-    elif admin_level == AdminLevel.ONE:
-        query = filter_is_unspecified(query, AvailabilityView.admin1_name, negate=True)
-        query = filter_is_unspecified(query, AvailabilityView.admin2_name)
-    elif admin_level == AdminLevel.TWO:
-        query = filter_is_unspecified(query, AvailabilityView.admin1_name, negate=True)
-        query = filter_is_unspecified(query, AvailabilityView.admin2_name, negate=True)
+    # if admin_level == AdminLevel.ZERO:
+    #     query = filter_is_unspecified(query, AvailabilityView.admin1_name)
+    #     query = filter_is_unspecified(query, AvailabilityView.admin2_name)
+    # elif admin_level == AdminLevel.ONE:
+    #     query = filter_is_unspecified(query, AvailabilityView.admin1_name, negate=True)
+    #     query = filter_is_unspecified(query, AvailabilityView.admin2_name)
+    # elif admin_level == AdminLevel.TWO:
+    #     query = filter_is_unspecified(query, AvailabilityView.admin1_name, negate=True)
+    #     query = filter_is_unspecified(query, AvailabilityView.admin2_name, negate=True)
 
     query = apply_pagination(query, pagination_parameters)
     query = query.order_by(
@@ -85,10 +74,10 @@ async def availability_view_list(
 
     return availabilities
 
+
 def filter_is_unspecified(query: Select, column: Mapped[str], negate=False) -> Select:
     or_clause = or_(column == '', column.is_(None), column.ilike(_UNSPECIFIED))
     if negate:
         return query.where(~or_clause)
     else:
         return query.where(or_clause)
-

hdx_hapi/db/dao/poverty_rate_dao.py

@@ -1,32 +1,42 @@
-from typing import Optional
+import logging
+from typing import Optional, Sequence
 
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy import select
 
 from hdx_hapi.db.models.views.vat_or_view import PovertyRateView
 from hdx_hapi.db.dao.util.util import (
     apply_date_range_filter,
+    apply_location_admin_1_filter,
     apply_pagination,
     apply_reference_period_filter,
-    case_insensitive_filter,
 )
-from hdx_hapi.endpoints.util.util import CommonDateRangeParams, PaginationParams, ReferencePeriodParameters
+from hdx_hapi.endpoints.util.util import (
+    CommonDateRangeParams,
+    CommonLocationAdm1Parameters,
+    PaginationParams,
+    ReferencePeriodParameters,
+)
+
+logger = logging.getLogger(__name__)
 
 
 async def poverty_rates_view_list(
     pagination_parameters: PaginationParams,
     common_date_range_params: CommonDateRangeParams,
     ref_period_parameters: Optional[ReferencePeriodParameters],
+    common_location_params: CommonLocationAdm1Parameters,
     db: AsyncSession,
     mpi_min: Optional[float] = None,
     mpi_max: Optional[float] = None,
-    location_ref: Optional[int] = None,
-    location_code: Optional[str] = None,
-    location_name: Optional[str] = None,
     has_hrp: Optional[bool] = None,
     in_gho: Optional[bool] = None,
-    provider_admin1_name: Optional[str] = None,
-):
+) -> Sequence[PovertyRateView]:
+    logger.info(
+        f'location_name={common_location_params.location_name}, '
+        f'admin1_code={common_location_params.admin1_code}, admin1_name={common_location_params.admin1_name}, '
+        f'ref_period_parameters={ref_period_parameters}, admin_level={common_location_params.admin_level}, '
+    )
     query = select(PovertyRateView)
 
     if mpi_min:
@@ -39,21 +49,20 @@ async def poverty_rates_view_list(
     if in_gho is not None:
         query = query.where(PovertyRateView.in_gho == in_gho)
 
-    if location_ref:
-        query = query.where(PovertyRateView.location_ref == location_ref)
-    if location_code:
-        query = case_insensitive_filter(query, PovertyRateView.location_code, location_code)
-    if location_name:
-        query = query.where(PovertyRateView.location_name.icontains(location_name))
-    if provider_admin1_name:
-        query = query.where(PovertyRateView.provider_admin1_name.icontains(provider_admin1_name))
-
     query = apply_date_range_filter(
         query,
         PovertyRateView,
         common_date_range_params,
     )
 
+    query = apply_location_admin_1_filter(
+        query,
+        PovertyRateView,
+        common_location_params,
+        has_hrp,
+        in_gho,
+    )
+
     query = apply_reference_period_filter(query, ref_period_parameters, PovertyRateView)
 
     query = apply_pagination(query, pagination_parameters)