Merge pull request #48 from OCHA-DAP/main

HDXDSYS-1537 HAPI Pipelines - fix failure for empty reportingDate in IDPs data

Author: ccataalin

requirements.txt

@@ -2,27 +2,27 @@
 #    uv pip compile pyproject.toml --resolver=backtracking --all-extras -o requirements.txt
 annotated-types==0.7.0
     # via pydantic
-attrs==24.2.0
+attrs==24.3.0
     # via
     #   frictionless
     #   jsonlines
     #   jsonschema
     #   referencing
 cachetools==5.5.0
     # via google-auth
-certifi==2024.8.30
+certifi==2024.12.14
     # via requests
 cfgv==3.4.0
     # via pre-commit
 chardet==5.2.0
     # via frictionless
-charset-normalizer==3.4.0
+charset-normalizer==3.4.1
     # via requests
 ckanapi==4.8
     # via hdx-python-api
-click==8.1.7
+click==8.1.8
     # via typer
-coverage==7.6.9
+coverage==7.6.10
     # via pytest-cov
 defopt==6.4.0
     # via hdx-python-api
@@ -44,7 +44,7 @@ filelock==3.16.1
     # via virtualenv
 frictionless==5.18.0
     # via hdx-python-utilities
-google-auth==2.36.0
+google-auth==2.37.0
     # via
     #   google-auth-oauthlib
     #   gspread
@@ -77,21 +77,21 @@ hdx-python-utilities==3.7.4
     #   hdx-python-scraper
 humanize==4.11.0
     # via frictionless
-identify==2.6.3
+identify==2.6.5
     # via pre-commit
 idna==3.10
     # via
     #   email-validator
     #   requests
 ijson==3.3.0
     # via hdx-python-utilities
-inflect==7.4.0
+inflect==7.5.0
     # via quantulum3
 iniconfig==2.0.0
     # via pytest
 isodate==0.7.2
     # via frictionless
-jinja2==3.1.4
+jinja2==3.1.5
     # via frictionless
 jsonlines==4.0.0
     # via hdx-python-utilities
@@ -124,7 +124,7 @@ more-itertools==10.5.0
     # via inflect
 nodeenv==1.9.1
     # via pre-commit
-num2words==0.5.13
+num2words==0.5.14
     # via quantulum3
 oauthlib==3.2.2
     # via requests-oauthlib
@@ -156,11 +156,11 @@ pyasn1==0.6.1
     #   rsa
 pyasn1-modules==0.4.1
     # via google-auth
-pydantic==2.10.3
+pydantic==2.10.4
     # via frictionless
-pydantic-core==2.27.1
+pydantic-core==2.27.2
     # via pydantic
-pygments==2.18.0
+pygments==2.19.0
     # via rich
 pyphonetics==0.5.3
     # via hdx-python-country
@@ -224,19 +224,19 @@ rpds-py==0.22.3
     #   referencing
 rsa==4.9
     # via google-auth
-ruamel-yaml==0.18.6
+ruamel-yaml==0.18.9
     # via hdx-python-utilities
 ruamel-yaml-clib==0.2.12
     # via ruamel-yaml
-setuptools==75.6.0
+setuptools==75.7.0
     # via ckanapi
 shellingham==1.5.4
     # via typer
 simpleeval==1.0.3
     # via frictionless
 simplejson==3.19.3
     # via ckanapi
-six==1.16.0
+six==1.17.0
     # via
     #   ckanapi
     #   pockets
@@ -278,13 +278,13 @@ unidecode==1.3.8
     # via
     #   libhxl
     #   pyphonetics
-urllib3==2.2.3
+urllib3==2.3.0
     # via
     #   libhxl
     #   requests
 validators==0.34.0
     # via frictionless
-virtualenv==20.28.0
+virtualenv==20.28.1
     # via pre-commit
 wheel==0.45.1
     # via libhxl

src/hapi/pipelines/database/idps.py

@@ -70,6 +70,17 @@ def populate(self) -> None:
                     date_reported = row[hxl_tags.index("#date+reported")]
                     reporting_round = row[hxl_tags.index("#round+code")]
                     operation = row[hxl_tags.index("#operation+name")]
+                    # Ignore rows with no reporting date since it is part of
+                    # the primary key of DBIDPs
+                    if date_reported is None:
+                        text = (
+                            f"No reportingDate for admin code {admin_code}, assessment type {assessment_type}, "
+                            f"reporting round {reporting_round}, operation {operation}"
+                        )
+                        self._error_manager.add_message(
+                            "IDPs", dataset_name, text
+                        )
+                        continue
                     duplicate_row_check = (
                         admin2_ref,
                         assessment_type,