Merge pull request #302 from OCHA-DAP/feature/hdx-10489-updated

Feature/hdx 10489 updated

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "src/hapi-schema"]
-path = src/hapi-schema
-url = https://github.com/OCHA-DAP/hapi-sqlalchemy-schema.git

.vscode/launch.json

@@ -12,6 +12,15 @@
             "console": "integratedTerminal",
             "justMyCode": true,
             "env":{"HAPI_USE_VAT": "false"}
+        },
+        {
+            "name": "generate-views",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "/srv/hapi/src/hapi-schema/src/hapi_schema/utils/hapi_views_code_generator.py",
+            "console": "integratedTerminal",
+            "justMyCode": true,
+            "env":{"HAPI_USE_VAT": "false"}
         }
     ],
 

hdx_hapi/db/dao/availability_view_dao.py

@@ -6,26 +6,21 @@
 from sqlalchemy.orm import Mapped
 
 from hdx_hapi.db.models.views.vat_or_view import AvailabilityView
-from hdx_hapi.db.dao.util.util import apply_pagination, case_insensitive_filter
-from hdx_hapi.endpoints.util.util import AdminLevel, PaginationParams
+from hdx_hapi.db.dao.util.util import apply_location_admin_filter, apply_pagination, case_insensitive_filter
+from hdx_hapi.endpoints.util.util import CommonLocationParameters, PaginationParams
 
 logger = logging.getLogger(__name__)
 _UNSPECIFIED = 'UNSPECIFIED'
 
+
 async def availability_view_list(
     pagination_parameters: PaginationParams,
+    common_location_params: CommonLocationParameters,
     db: AsyncSession,
     category: Optional[str] = None,
     subcategory: Optional[str] = None,
-    location_name: Optional[str] = None,
-    location_code: Optional[str] = None,
-    admin1_name: Optional[str] = None,
-    admin1_code: Optional[str] = None,
-    admin2_name: Optional[str] = None,
-    admin2_code: Optional[str] = None,
     hapi_updated_date_min: Optional[datetime.datetime | datetime.date] = None,
     hapi_updated_date_max: Optional[datetime.datetime | datetime.date] = None,
-    admin_level: Optional[AdminLevel] = None,
 ):
     logger.info(f'availability_view_list called with params: {locals()}')
 
@@ -35,34 +30,28 @@ async def availability_view_list(
     if subcategory:
         query = case_insensitive_filter(query, AvailabilityView.subcategory, subcategory)
 
-    if location_code:
-        query = case_insensitive_filter(query, AvailabilityView.location_code, location_code)
-    if location_name:
-        query = query.where(AvailabilityView.location_name.icontains(location_name))
-    if admin1_code:
-        query = case_insensitive_filter(query, AvailabilityView.admin1_code, admin1_code)
-    if admin1_name:
-        query = query.where(AvailabilityView.admin1_name.icontains(admin1_name))
-    if admin2_code:
-        query = case_insensitive_filter(query, AvailabilityView.admin2_code, admin2_code)
-    if admin2_name:
-        query = query.where(AvailabilityView.admin2_name.icontains(admin2_name))
+    query = apply_location_admin_filter(
+        query,
+        AvailabilityView,
+        common_location_params,
+    )
+
     if hapi_updated_date_min:
         query = query.where(AvailabilityView.hapi_updated_date >= hapi_updated_date_min)
     if hapi_updated_date_max:
         query = query.where(AvailabilityView.hapi_updated_date < hapi_updated_date_max)
 
     # Admin level filtering. Filtering by admin level is handled differently for the data availability table
     # beause we don't have the adminX_is_unspecified fields.
-    if admin_level == AdminLevel.ZERO:
-        query = filter_is_unspecified(query, AvailabilityView.admin1_name)
-        query = filter_is_unspecified(query, AvailabilityView.admin2_name)
-    elif admin_level == AdminLevel.ONE:
-        query = filter_is_unspecified(query, AvailabilityView.admin1_name, negate=True)
-        query = filter_is_unspecified(query, AvailabilityView.admin2_name)
-    elif admin_level == AdminLevel.TWO:
-        query = filter_is_unspecified(query, AvailabilityView.admin1_name, negate=True)
-        query = filter_is_unspecified(query, AvailabilityView.admin2_name, negate=True)
+    # if admin_level == AdminLevel.ZERO:
+    #     query = filter_is_unspecified(query, AvailabilityView.admin1_name)
+    #     query = filter_is_unspecified(query, AvailabilityView.admin2_name)
+    # elif admin_level == AdminLevel.ONE:
+    #     query = filter_is_unspecified(query, AvailabilityView.admin1_name, negate=True)
+    #     query = filter_is_unspecified(query, AvailabilityView.admin2_name)
+    # elif admin_level == AdminLevel.TWO:
+    #     query = filter_is_unspecified(query, AvailabilityView.admin1_name, negate=True)
+    #     query = filter_is_unspecified(query, AvailabilityView.admin2_name, negate=True)
 
     query = apply_pagination(query, pagination_parameters)
     query = query.order_by(
@@ -85,10 +74,10 @@ async def availability_view_list(
 
     return availabilities
 
+
 def filter_is_unspecified(query: Select, column: Mapped[str], negate=False) -> Select:
     or_clause = or_(column == '', column.is_(None), column.ilike(_UNSPECIFIED))
     if negate:
         return query.where(~or_clause)
     else:
         return query.where(or_clause)
-

hdx_hapi/db/dao/poverty_rate_dao.py

@@ -1,32 +1,42 @@
-from typing import Optional
+import logging
+from typing import Optional, Sequence
 
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy import select
 
 from hdx_hapi.db.models.views.vat_or_view import PovertyRateView
 from hdx_hapi.db.dao.util.util import (
     apply_date_range_filter,
+    apply_location_admin_1_filter,
     apply_pagination,
     apply_reference_period_filter,
-    case_insensitive_filter,
 )
-from hdx_hapi.endpoints.util.util import CommonDateRangeParams, PaginationParams, ReferencePeriodParameters
+from hdx_hapi.endpoints.util.util import (
+    CommonDateRangeParams,
+    CommonLocationAdm1Parameters,
+    PaginationParams,
+    ReferencePeriodParameters,
+)
+
+logger = logging.getLogger(__name__)
 
 
 async def poverty_rates_view_list(
     pagination_parameters: PaginationParams,
     common_date_range_params: CommonDateRangeParams,
     ref_period_parameters: Optional[ReferencePeriodParameters],
+    common_location_params: CommonLocationAdm1Parameters,
     db: AsyncSession,
     mpi_min: Optional[float] = None,
     mpi_max: Optional[float] = None,
-    location_ref: Optional[int] = None,
-    location_code: Optional[str] = None,
-    location_name: Optional[str] = None,
     has_hrp: Optional[bool] = None,
     in_gho: Optional[bool] = None,
-    provider_admin1_name: Optional[str] = None,
-):
+) -> Sequence[PovertyRateView]:
+    logger.info(
+        f'location_name={common_location_params.location_name}, '
+        f'admin1_code={common_location_params.admin1_code}, admin1_name={common_location_params.admin1_name}, '
+        f'ref_period_parameters={ref_period_parameters}, admin_level={common_location_params.admin_level}, '
+    )
     query = select(PovertyRateView)
 
     if mpi_min:
@@ -39,21 +49,20 @@ async def poverty_rates_view_list(
     if in_gho is not None:
         query = query.where(PovertyRateView.in_gho == in_gho)
 
-    if location_ref:
-        query = query.where(PovertyRateView.location_ref == location_ref)
-    if location_code:
-        query = case_insensitive_filter(query, PovertyRateView.location_code, location_code)
-    if location_name:
-        query = query.where(PovertyRateView.location_name.icontains(location_name))
-    if provider_admin1_name:
-        query = query.where(PovertyRateView.provider_admin1_name.icontains(provider_admin1_name))
-
     query = apply_date_range_filter(
         query,
         PovertyRateView,
         common_date_range_params,
     )
 
+    query = apply_location_admin_1_filter(
+        query,
+        PovertyRateView,
+        common_location_params,
+        has_hrp,
+        in_gho,
+    )
+
     query = apply_reference_period_filter(query, ref_period_parameters, PovertyRateView)
 
     query = apply_pagination(query, pagination_parameters)

hdx_hapi/db/dao/util/util.py

@@ -1,12 +1,13 @@
 import datetime
 from typing import Optional, Protocol, Type
-from sqlalchemy import Select, or_, and_
+from sqlalchemy import Select, or_
 from sqlalchemy.orm import Mapped
 
 from hdx_hapi.config.config import get_config
 from hdx_hapi.endpoints.util.util import (
     AdminLevel,
     CommonDateRangeParams,
+    CommonLocationAdm1Parameters,
     CommonLocationParameters,
     PaginationParams,
     ReferencePeriodParameters,
@@ -89,74 +90,103 @@ class EntityWithLocationAdmin(Protocol):
     admin2_name: Mapped[str]
     provider_admin2_name: Mapped[str]
     admin2_is_unspecified: Mapped[bool]
+    admin_level: Mapped[int]
 
 
-def apply_location_admin_filter(
+def _apply_location_admin_filter(
     query: Select,
     db_class: Type[EntityWithLocationAdmin],
-    common_location_parameters: CommonLocationParameters,
     has_hrp: Optional[bool] = None,
     in_gho: Optional[bool] = None,
+    location_code: Optional[str] = None,
+    location_name: Optional[str] = None,
+    admin1_code: Optional[str] = None,
+    admin1_name: Optional[str] = None,
+    admin2_code: Optional[str] = None,
+    admin2_name: Optional[str] = None,
+    admin_level: Optional[AdminLevel] = None,
 ) -> Select:
-    location_ref = common_location_parameters.location_ref
-    location_code = common_location_parameters.location_code
-    location_name = common_location_parameters.location_name
-    admin1_ref = common_location_parameters.admin1_ref
-    admin1_code = common_location_parameters.admin1_code
-    admin1_name = common_location_parameters.admin1_name
-    provider_admin1_name = common_location_parameters.provider_admin1_name
-    admin2_ref = common_location_parameters.admin2_ref
-    admin2_code = common_location_parameters.admin2_code
-    admin2_name = common_location_parameters.admin2_name
-    provider_admin2_name = common_location_parameters.provider_admin2_name
-    admin_level = common_location_parameters.admin_level
-
-    if location_ref:
-        query = query.where(db_class.location_ref == location_ref)
     if location_code:
         query = case_insensitive_filter(query, db_class.location_code, location_code)
     if location_name:
         query = query.where(db_class.location_name.icontains(location_name))
-    if admin1_ref:
-        query = query.where(db_class.admin1_ref == admin1_ref)
     if admin1_code:
         query = case_insensitive_filter(query, db_class.admin1_code, admin1_code)
     if admin1_name:
         query = query.where(db_class.admin1_name.icontains(admin1_name))
-    # if provider_admin1_name_is_unspecified:
-    #     query = query.where(or_(db_class.provider_admin1_name == '', db_class.provider_admin1_name.is_(None)))
-    if provider_admin1_name:
-        query = query.where(db_class.provider_admin1_name.icontains(provider_admin1_name))
-    if admin2_ref:
-        query = query.where(db_class.admin2_ref == admin2_ref)
     if admin2_code:
         query = case_insensitive_filter(query, db_class.admin2_code, admin2_code)
     if admin2_name:
         query = query.where(db_class.admin2_name.icontains(admin2_name))
-    # if provider_admin2_name_is_unspecified:
-    #     query = query.where(or_(db_class.provider_admin2_name == '', db_class.provider_admin2_name.is_(None)))
-    if provider_admin2_name:
-        query = query.where(db_class.provider_admin2_name.icontains(provider_admin2_name))
-    # if admin1_is_unspecified is not None:
-    #     query = query.where(db_class.admin1_is_unspecified == admin1_is_unspecified)
-    # if admin2_is_unspecified is not None:
-    #     query = query.where(db_class.admin2_is_unspecified == admin2_is_unspecified)
     if has_hrp is not None:
         query = query.where(db_class.has_hrp == has_hrp)
     if in_gho is not None:
         query = query.where(db_class.in_gho == in_gho)
 
-    if AdminLevel.TWO == admin_level:
-        query = query.where(or_(db_class.admin2_is_unspecified == False, db_class.provider_admin2_name != ''))
-    elif AdminLevel.ONE == admin_level:
-        query = query.where(and_(db_class.admin2_is_unspecified == True, db_class.provider_admin2_name == ''))
-        query = query.where(or_(db_class.admin1_is_unspecified == False, db_class.provider_admin1_name != ''))
-    elif AdminLevel.ZERO == admin_level:
-        query = query.where(and_(db_class.admin1_is_unspecified == True, db_class.provider_admin1_name == ''))
+    if admin_level is not None:
+        query = query.where(db_class.admin_level == int(admin_level.value))
 
     return query
 
 
+def apply_location_admin_filter(
+    query: Select,
+    db_class: Type[EntityWithLocationAdmin],
+    common_location_parameters: CommonLocationParameters,
+    has_hrp: Optional[bool] = None,
+    in_gho: Optional[bool] = None,
+) -> Select:
+    location_code = common_location_parameters.location_code
+    location_name = common_location_parameters.location_name
+    admin1_code = common_location_parameters.admin1_code
+    admin1_name = common_location_parameters.admin1_name
+    admin2_code = common_location_parameters.admin2_code
+    admin2_name = common_location_parameters.admin2_name
+    admin_level = common_location_parameters.admin_level
+
+    return _apply_location_admin_filter(
+        query=query,
+        db_class=db_class,
+        has_hrp=has_hrp,
+        in_gho=in_gho,
+        location_code=location_code,
+        location_name=location_name,
+        admin1_code=admin1_code,
+        admin1_name=admin1_name,
+        admin2_code=admin2_code,
+        admin2_name=admin2_name,
+        admin_level=admin_level,
+    )
+
+
+def apply_location_admin_1_filter(
+    query: Select,
+    db_class: Type[EntityWithLocationAdmin],
+    common_location_parameters: CommonLocationAdm1Parameters,
+    has_hrp: Optional[bool] = None,
+    in_gho: Optional[bool] = None,
+) -> Select:
+    location_code = common_location_parameters.location_code
+    location_name = common_location_parameters.location_name
+    admin1_code = common_location_parameters.admin1_code
+    admin1_name = common_location_parameters.admin1_name
+    admin_level = common_location_parameters.admin_level
+
+    return _apply_location_admin_filter(
+        query=query,
+        db_class=db_class,
+        has_hrp=has_hrp,
+        in_gho=in_gho,
+        location_code=location_code,
+        location_name=location_name,
+        admin1_code=admin1_code,
+        admin1_name=admin1_name,
+        admin2_code=None,
+        admin2_name=None,
+        admin_level=admin_level,
+    )
+
+
 def case_insensitive_filter(query: Select, column: Mapped[str], value: str) -> Select:
     query = query.where(column.ilike(value))
     return query

hdx_hapi/db/models/views/util/util.py

@@ -5,6 +5,7 @@
 
 # This is based on https://github.com/sqlalchemy/sqlalchemy/wiki/Views
 
+
 class CreateView(DDLElement):
     def __init__(self, name, selectable):
         self.name = name
@@ -41,15 +42,13 @@ def view(name, metadata, selectable):
     t = table(name)
 
     t._columns._populate_separate_keys(
-        col._make_proxy(t) for col in selectable.selected_columns
+        col._make_proxy(t, primary_key=set(), foreign_keys=set()) for col in selectable.selected_columns
     )
 
     sa.event.listen(
         metadata,
         'after_create',
         CreateView(name, selectable).execute_if(callable_=view_doesnt_exist),
     )
-    sa.event.listen(
-        metadata, 'before_drop', DropView(name).execute_if(callable_=view_exists)
-    )
-    return t
+    sa.event.listen(metadata, 'before_drop', DropView(name).execute_if(callable_=view_exists))
+    return t