Merge pull request #8 from OCHA-DAP/main

New 3W data and remove phonetic matching

Author: alexandru-m-g

CHANGELOG.md

@@ -4,7 +4,31 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
-### [0.9.20] - 2024-06-29
+## [0.9.23] - 2024-07-05
+
+### Changed
+
+- Use normalise function from HDX Python Utilities
+- Update mappings for changes in HDX Python Country
+
+## [0.9.22] - 2024-07-05
+
+### Fixed
+
+- Use latest operational presence data for NER and SDN
+
+## [0.9.21] - 2024-07-05
+
+### Changed
+
+- Added parameter to make phonetic matching optional for org types and sectors
+- Added clean_text function to utilities
+
+### Fixed
+
+- Removed outdated error messages from operational presence pipeline
+
+## [0.9.20] - 2024-06-29
 
 ### Fixed
 

pyproject.toml

@@ -34,12 +34,12 @@ classifiers = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "hapi-schema>=0.8.9",
+    "hapi-schema>=0.8.12",
     "hdx-python-api>= 6.2.9",
-    "hdx-python-country>= 3.7.4",
+    "hdx-python-country>= 3.7.5",
     "hdx-python-database[postgresql]>= 1.3.1",
-    "hdx-python-scraper>= 2.3.7",
-    "hdx-python-utilities>= 3.6.9",
+    "hdx-python-scraper>= 2.3.8",
+    "hdx-python-utilities>= 3.7.2",
     "libhxl",
     "sqlalchemy"
 ]

requirements.txt

@@ -10,7 +10,7 @@ attrs==23.2.0
     #   referencing
 cachetools==5.3.3
     # via google-auth
-certifi==2024.6.2
+certifi==2024.7.4
     # via requests
 cffi==1.16.0
     # via cryptography
@@ -48,7 +48,7 @@ filelock==3.15.4
     # via virtualenv
 frictionless==5.17.0
     # via hdx-python-utilities
-google-auth==2.30.0
+google-auth==2.31.0
     # via
     #   google-auth-oauthlib
     #   gspread
@@ -58,13 +58,13 @@ greenlet==3.0.3
     # via sqlalchemy
 gspread==6.1.2
     # via hdx-python-scraper
-hapi-schema==0.8.12
+hapi-schema==0.8.13
     # via hapi-pipelines (pyproject.toml)
 hdx-python-api==6.3.1
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hdx-python-scraper
-hdx-python-country==3.7.4
+hdx-python-country==3.7.5
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hdx-python-api
@@ -73,7 +73,7 @@ hdx-python-database==1.3.1
     # via hapi-pipelines (pyproject.toml)
 hdx-python-scraper==2.3.8
     # via hapi-pipelines (pyproject.toml)
-hdx-python-utilities==3.7.0
+hdx-python-utilities==3.7.2
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hdx-python-api
@@ -89,7 +89,7 @@ idna==3.7
     #   requests
 ijson==3.3.0
     # via hdx-python-utilities
-inflect==7.3.0
+inflect==7.3.1
     # via quantulum3
 iniconfig==2.0.0
     # via pytest
@@ -152,9 +152,9 @@ pockets==0.9.1
     # via sphinxcontrib-napoleon
 pre-commit==3.7.1
     # via hapi-pipelines (pyproject.toml)
-psycopg==3.1.19
+psycopg==3.2.1
     # via hdx-python-database
-psycopg-binary==3.1.19
+psycopg-binary==3.2.1
     # via psycopg
 pyasn1==0.6.0
     # via
@@ -166,9 +166,9 @@ pyasn1-modules==0.4.0
     # via google-auth
 pycparser==2.22
     # via cffi
-pydantic==2.7.4
+pydantic==2.8.2
     # via frictionless
-pydantic-core==2.18.4
+pydantic-core==2.20.1
     # via pydantic
 pygments==2.18.0
     # via rich
@@ -242,7 +242,7 @@ ruamel-yaml==0.18.6
     # via hdx-python-utilities
 ruamel-yaml-clib==0.2.8
     # via ruamel-yaml
-setuptools==70.1.1
+setuptools==70.2.0
     # via ckanapi
 shellingham==1.5.4
     # via typer
@@ -295,7 +295,7 @@ urllib3==2.2.2
     # via
     #   libhxl
     #   requests
-validators==0.28.3
+validators==0.30.0
     # via frictionless
 virtualenv==20.26.3
     # via pre-commit

src/hapi/pipelines/configs/core.yaml

@@ -55,6 +55,7 @@ admin2:
     "AF08|Onaba": "AF0805"
     "AF14|Khost": "AF1401"
     "AF23|Chaghcharan": "AF2301"
+    "CM008|Ndé": "CM008007"
     "CO08|Distrito Especial, Industrial Y Portuario De Barranquilla": "CO08001"
     "ET01|C. TIGRAY": "ET0102"
     "ET01|NW. TIGRAY": "ET0101"
@@ -187,16 +188,17 @@ admin2:
     "UA23|Vilnianskyi": "UA2306"
     "UA23|Yakymivskyi": "UA2308"
     "YE14|Radman Al Awad": "YE1412"
+    "YE21|Ain": "YE2106"
     "YE24|Craiter": "YE2407"
     "YE26|Medghal": "YE2603"
 
   admin_name_replacements:
     "COD| city": ""
-    "ETH|c.": "central"
-    "ETH|e.": "east"
-    "ETH|n.": "north"
-    "ETH|s.": "south"
-    "ETH|w.": "west"
+    "ETH|c ": "central "
+    "ETH|e ": "east "
+    "ETH|n ": "north "
+    "ETH|s ": "south "
+    "ETH|w ": "west "
 
 orphan_admin2s:
   SS0001: "SSD"
@@ -232,6 +234,7 @@ org_type_map:
   ingo: "437"
   institution etatique: "435"
   international institution: "438"
+  international ngos: "437"
   local ngo: "504"
   mouv cr: "445"
   mouvement croix rouge: "445"
@@ -250,6 +253,7 @@ org_type_map:
   ong nationale: "441"
   organisation des nations unies: "447"
   organismo internacional: "438"
+  others: "443"
   otro: "443"
   programme de developpement: "503"
   red cross: "445"
@@ -270,31 +274,30 @@ sector_map:
   abri bna: "SHL"
   abris: "SHL"
   abris ame: "SHL"
-  abris/ame: "SHL"
-  abris/bna: "SHL"
-  abris/bna/cccm: "SHL"
-  abris / nfi: "SHL"
+  abris bna: "SHL"
+  abris bna cccm: "SHL"
   abris d'urgence et nfi: "SHL"
+  abris nfi: "SHL"
   action contre les mines: "PRO-MIN"
   aee: "SHL"
   agriculture: "FSC"
   agua saneamiento e higiene: "WSH"
   all: "Intersectoral"
-  ame: "SHL"
   alojamiento de emergencia: "SHL"
-  alojamiento de emergencia (shelter): "SHL"
+  alojamiento de emergencia shelter: "SHL"
   alojamientos y asentamientos: "SHL"
+  ame: "SHL"
   ash: "WSH"
   assainissement: "WSH"
-  camp coordination camp management: "CCM"
   camp coordination and camp management: "CCM"
+  camp coordination camp management: "CCM"
   cash: "Cash"
   cccm: "CCM"
   ccs: "CCM"
   cluster coordination: "CCM"
-  coord & services support: "CCM"
+  coord services support: "CCM"
+  coordinacion informacion: "CCM"
   coordination: "CCM"
-  coordinacion / informacion: "CCM"
   coordination et gestion des camps: "CCM"
   eah: "WSH"
   eau: "WSH"
@@ -303,8 +306,8 @@ sector_map:
   eau hygiene assainissement: "WSH"
   eau hygiene et assainissement: "WSH"
   educacion: "EDU"
-  education: "EDU"
   educacion en emergencias: "EDU"
+  education: "EDU"
   eha: "WSH"
   emergency shelter and non food items: "SHL"
   epah: "WSH"
@@ -318,6 +321,7 @@ sector_map:
   food security livelihood: "FSC"
   fsl: "FSC"
   gestion des sites d'accueil temporaires: "SHL"
+  gestion des sites d accueil temporaires: "SHL"
   gbv: "PRO-GBV"
   hlp: "PRO-HLP"
   humanitaire: "Hum"
@@ -333,13 +337,14 @@ sector_map:
   multisectoriel: "Multi"
   nutricion: "NUT"
   nutrition: "NUT"
-  operatioanl presence water sanitation & hygiene: "WSH"
+  operatioanl presence water sanitation hygiene: "WSH"
   operational presence education in emergencies: "EDU"
-  operational presence emergency shelter & non food items: "SHL"
-  operational presence food security & agriculture: "FSC"
+  operational presence emergency shelter non food items: "SHL"
+  operational presence food security agriculture: "FSC"
   operational presence health: "HEA"
   operational presence nutrition: "NUT"
   operational presence protection: "PRO"
+  pro cpm: "PRO-CPN"
   pronna: "PRO-CPN"
   propg: "PRO"
   proteccion infantil: "PRO-CPN"
@@ -350,6 +355,7 @@ sector_map:
   protection logement terre et propriete: "PRO-HLP"
   protection ltb: "PRO-HLP"
   protection lutte anti mines: "PRO-MIN"
+  protection pe: "PRO-CPN"
   protection protection de l'enfant: "PRO-CPN"
   protection violences basees sur le genre: "PRO-GBV"
   protection vgb: "PRO-GBV"
@@ -363,10 +369,10 @@ sector_map:
   rcf health and nutrtion: "HEA"
   rcf protection: "PRO"
   recuperacion temprana: "ERY"
-  relevement rapide: "ERY"
   relevement precoce: "ERY"
+  relevement rapide: "ERY"
   refugee response: "CCM"
-  refugees & migrants multi sector: "CCM"
+  refugees migrants multi sector: "CCM"
   reponse aux refugies: "CCM"
   sa: "FSC"
   sal: "HEA"
@@ -379,18 +385,19 @@ sector_map:
   services humanitaires communs: "Hum"
   sexual and reproductive health: "HEA"
   shelter: "SHL"
+  shelter nfi: "SHL"
   shelter nfis: "SHL"
-  shelter/nfi: "SHL"
   shelter and nfi: "SHL"
   shelter and nfis: "SHL"
   shelter and non food items: "SHL"
   site management: "CCM"
   snfi: "SHL"
   telecommunications: "TEL"
-  telecomunicaciones de emergencia: "TEL"
   telecommunications d'urgence: "TEL"
+  telecomunicaciones de emergencia: "TEL"
   vbg: "PRO-GBV"
   violences basees sur le genre: "PRO-GBV"
+  violence basee sur le genre: "PRO-GBV"
   violencia basada en genero: "PRO-GBV"
   wash: "WSH"
   water sanitation and hygiene: "WSH"

src/hapi/pipelines/configs/operational_presence.yaml

@@ -1,7 +1,4 @@
 #Operational presence config file
-operational_presence_error_messages:
-  burkina-faso-presence-operationnelle: "Data file cannot be read"
-  drc_presence_operationnelle: "Data file cannot be read"
 
 operational_presence_default:
   scrapers_with_defaults:
@@ -461,15 +458,15 @@ operational_presence_national:
 
   operational_presence_ner:
     dataset: "niger-operational-presence"
-    resource: "NER_Sep_2023"
+    resource: "3w_OP_Nationale_Juin 2024 au 08062024.xlsx"
     format: "xlsx"
-    sheet: "3W_Data_T2"
+    sheet: "3w_OP_Nationale_compilation"
     headers:
-      - 4
-      - 5
+      - 1
+      - 2
     source_date:
-      start: "01/07/2023"
-      end: "30/09/2023"
+      start: "01/06/2024"
+      end: "08/06/2024"
     admin_single: "NER"
     input:
       - "NOM ORGANISATION
@@ -478,21 +475,19 @@ operational_presence_national:
         (Choisir dans la liste déroulante)
         #org+acronym"
       - "TYPE ORGANISATION
-        (cette colonne est renseignée automatiquement selon la selection dans le colonne B)"
+        (cette colonne est renseignée automatiquement selon la selection dans le colonne B"
       - "CLUSTER
-        (Choisir dans la liste déroulante)
-        #sector"
+        (Choisir dans la liste déroulante) Pour les projets humanitaires uniquement"
     list:
       - "NOM ORGANISATION
         (cette colonne est renseignée automatiquement selon la selection dans le colonne A)"
       - "ACCRONYME
         (Choisir dans la liste déroulante)
         #org+acronym"
       - "TYPE ORGANISATION
-        (cette colonne est renseignée automatiquement selon la selection dans le colonne B)"
+        (cette colonne est renseignée automatiquement selon la selection dans le colonne B"
       - "CLUSTER
-        (Choisir dans la liste déroulante)
-        #sector"
+        (Choisir dans la liste déroulante) Pour les projets humanitaires uniquement"
 
   operational_presence_pse:
     dataset: "state-of-palestine-who-does-what-and-where-3ws"
@@ -517,13 +512,13 @@ operational_presence_national:
 
   operational_presence_sdn:
     dataset: "sudan-operational-presence"
-    resource: "2024-consolidated-3w-data-jan-to-31-march.xlsx"
+    resource: "2024-consolidated-3w-data-jan-to-31-may.xlsx"
     format: "xlsx"
     sheet: "3W_master_data"
     headers: 1
     source_date:
       start: "01/01/2024"
-      end: "31/03/2024"
+      end: "31/05/2024"
     admin_single: "SDN"
     input:
       - "Organization"