Merge pull request #25 from OCHA-DAP/main

PR for merging hapi-pipelines into hapi-pipelines-prod

Author: alexandru-m-g

dump_views.sh

@@ -5,7 +5,7 @@ echo "Started at $now"
 
 mkdir -p database/csv
 
-views=$(docker exec -t postgres-container psql -t -U postgres -d hapi -c \
+views=$(docker exec -i postgres-container psql -t -U postgres -d hapi -c \
   "select table_name from INFORMATION_SCHEMA.views WHERE table_schema = ANY (current_schemas(false))")
 
 for view in $views

pyproject.toml

@@ -34,7 +34,7 @@ classifiers = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "hapi-schema>=0.8.14",
+    "hapi-schema>=0.8.15",
     "hdx-python-api>= 6.3.2",
     "hdx-python-country>= 3.7.7",
     "hdx-python-database[postgresql]>= 1.3.1",

requirements.txt

@@ -26,7 +26,7 @@ click==8.1.7
     # via typer
 coverage==7.6.1
     # via pytest-cov
-cryptography==43.0.0
+cryptography==43.0.1
     # via pyopenssl
 defopt==6.4.0
     # via hdx-python-api
@@ -46,7 +46,7 @@ et-xmlfile==1.1.0
     # via openpyxl
 filelock==3.15.4
     # via virtualenv
-frictionless==5.17.0
+frictionless==5.17.1
     # via hdx-python-utilities
 google-auth==2.34.0
     # via
@@ -58,7 +58,7 @@ greenlet==3.0.3
     # via sqlalchemy
 gspread==6.1.2
     # via hdx-python-scraper
-hapi-schema==0.8.14
+hapi-schema==0.8.16
     # via hapi-pipelines (pyproject.toml)
 hdx-python-api==6.3.2
     # via
@@ -242,7 +242,7 @@ ruamel-yaml==0.18.6
     # via hdx-python-utilities
 ruamel-yaml-clib==0.2.8
     # via ruamel-yaml
-setuptools==74.0.0
+setuptools==74.1.1
     # via ckanapi
 shellingham==1.5.4
     # via typer
@@ -259,7 +259,7 @@ six==1.16.0
     #   sphinxcontrib-napoleon
 sphinxcontrib-napoleon==0.7
     # via defopt
-sqlalchemy==2.0.32
+sqlalchemy==2.0.33
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hapi-schema
@@ -295,7 +295,7 @@ urllib3==2.2.2
     # via
     #   libhxl
     #   requests
-validators==0.33.0
+validators==0.34.0
     # via frictionless
 virtualenv==20.26.3
     # via pre-commit

src/hapi/pipelines/database/poverty_rate.py

@@ -1,7 +1,7 @@
 """Functions specific to the funding theme."""
 
 from collections import defaultdict
-from datetime import date
+from datetime import datetime
 from logging import getLogger
 from typing import Dict
 
@@ -107,11 +107,11 @@ def populate(self) -> None:
         self._session.commit()
 
 
-def _convert_year_to_reference_period(year: str) -> [date, date]:
+def _convert_year_to_reference_period(year: str) -> [datetime, datetime]:
     # The year column can either be a single year or a range split by a dash.
     # This function turns this into a reference period start and end date.
     try:
         start_year, end_year = year.split("-")
     except ValueError:
         start_year, end_year = year, year
-    return date(int(start_year), 1, 1), date(int(end_year), 12, 31)
+    return datetime(int(start_year), 1, 1), datetime(int(end_year), 12, 31)

tests/test_main.py

@@ -116,7 +116,7 @@ def pipelines(self, configuration, folder, themes_to_run):
                     count = session.scalar(select(func.count(DBAdmin1.id)))
                     check.equal(count, 703)
                     count = session.scalar(select(func.count(DBAdmin2.id)))
-                    check.equal(count, 6160)
+                    check.equal(count, 5942)
                     count = session.scalar(select(func.count(DBSector.code)))
                     check.equal(count, 19)
                     count = session.scalar(select(func.count(DBCurrency.code)))
@@ -257,6 +257,6 @@ def test_food_prices(self, configuration, folder, pipelines):
         count = session.scalar(select(func.count(DBWFPCommodity.code)))
         check.equal(count, 1077)
         count = session.scalar(select(func.count(DBWFPMarket.code)))
-        check.equal(count, 4100)
+        check.equal(count, 4040)
         count = session.scalar(select(func.count(DBFoodPrice.resource_hdx_id)))
         check.equal(count, 31615)