HDXDSYS-868 Use normalise function from HDX Python Utilities (OCHA-DAP#122)

* Use normalise function from HDX Python Utilities

* 3 letter names not matched now because we set a minimum length of 4, so add mappings
Fix name replacements

* Update CHANGELOG

Author: mcarans

CHANGELOG.md

@@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [0.9.23] - 2024-07-05
+
+### Changed
+
+- Use normalise function from HDX Python Utilities
+- Update mappings for changes in HDX Python Country
+
 ## [0.9.22] - 2024-07-05
 
 ### Fixed

pyproject.toml

@@ -34,12 +34,12 @@ classifiers = [
 requires-python = ">=3.8"
 
 dependencies = [
-    "hapi-schema>=0.8.9",
+    "hapi-schema>=0.8.12",
     "hdx-python-api>= 6.2.9",
-    "hdx-python-country>= 3.7.4",
+    "hdx-python-country>= 3.7.5",
     "hdx-python-database[postgresql]>= 1.3.1",
-    "hdx-python-scraper>= 2.3.7",
-    "hdx-python-utilities>= 3.6.9",
+    "hdx-python-scraper>= 2.3.8",
+    "hdx-python-utilities>= 3.7.2",
     "libhxl",
     "sqlalchemy"
 ]

requirements.txt

@@ -10,7 +10,7 @@ attrs==23.2.0
     #   referencing
 cachetools==5.3.3
     # via google-auth
-certifi==2024.6.2
+certifi==2024.7.4
     # via requests
 cffi==1.16.0
     # via cryptography
@@ -64,7 +64,7 @@ hdx-python-api==6.3.1
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hdx-python-scraper
-hdx-python-country==3.7.4
+hdx-python-country==3.7.5
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hdx-python-api
@@ -73,7 +73,7 @@ hdx-python-database==1.3.1
     # via hapi-pipelines (pyproject.toml)
 hdx-python-scraper==2.3.8
     # via hapi-pipelines (pyproject.toml)
-hdx-python-utilities==3.7.1
+hdx-python-utilities==3.7.2
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hdx-python-api
@@ -166,7 +166,7 @@ pyasn1-modules==0.4.0
     # via google-auth
 pycparser==2.22
     # via cffi
-pydantic==2.8.1
+pydantic==2.8.2
     # via frictionless
 pydantic-core==2.20.1
     # via pydantic
@@ -295,7 +295,7 @@ urllib3==2.2.2
     # via
     #   libhxl
     #   requests
-validators==0.29.0
+validators==0.30.0
     # via frictionless
 virtualenv==20.26.3
     # via pre-commit

src/hapi/pipelines/configs/core.yaml

@@ -55,6 +55,7 @@ admin2:
     "AF08|Onaba": "AF0805"
     "AF14|Khost": "AF1401"
     "AF23|Chaghcharan": "AF2301"
+    "CM008|Ndé": "CM008007"
     "CO08|Distrito Especial, Industrial Y Portuario De Barranquilla": "CO08001"
     "ET01|C. TIGRAY": "ET0102"
     "ET01|NW. TIGRAY": "ET0101"
@@ -187,16 +188,17 @@ admin2:
     "UA23|Vilnianskyi": "UA2306"
     "UA23|Yakymivskyi": "UA2308"
     "YE14|Radman Al Awad": "YE1412"
+    "YE21|Ain": "YE2106"
     "YE24|Craiter": "YE2407"
     "YE26|Medghal": "YE2603"
 
   admin_name_replacements:
     "COD| city": ""
-    "ETH|c.": "central"
-    "ETH|e.": "east"
-    "ETH|n.": "north"
-    "ETH|s.": "south"
-    "ETH|w.": "west"
+    "ETH|c ": "central "
+    "ETH|e ": "east "
+    "ETH|n ": "north "
+    "ETH|s ": "south "
+    "ETH|w ": "west "
 
 orphan_admin2s:
   SS0001: "SSD"

src/hapi/pipelines/database/operational_presence.py

@@ -7,11 +7,11 @@
 from hapi_schema.db_operational_presence import DBOperationalPresence
 from hdx.location.adminlevel import AdminLevel
 from hdx.utilities.dictandlist import write_list_to_csv
+from hdx.utilities.text import normalise
 from sqlalchemy.orm import Session
 
 from ..utilities.batch_populate import batch_populate
 from ..utilities.logging_helpers import add_message, add_missing_value_message
-from ..utilities.mappings import clean_text
 from . import admins
 from .base_uploader import BaseUploader
 from .metadata import Metadata
@@ -160,8 +160,8 @@ def populate(self, debug=False):
                         )
                         org_acronym, org_name, org_type = self._org.data[
                             (
-                                clean_text(org_acronym),
-                                clean_text(org_name),
+                                normalise(org_acronym),
+                                normalise(org_name),
                             )
                         ]
                         sector_code = self._sector.get_sector_code(sector_orig)

src/hapi/pipelines/database/org.py

@@ -6,10 +6,10 @@
 from hapi_schema.db_org import DBOrg
 from hdx.scraper.utilities.reader import Read
 from hdx.utilities.dictandlist import dict_of_sets_add
+from hdx.utilities.text import normalise
 from sqlalchemy.orm import Session
 
 from ..utilities.batch_populate import batch_populate
-from ..utilities.mappings import clean_text
 from .base_uploader import BaseUploader
 
 logger = logging.getLogger(__name__)
@@ -57,8 +57,8 @@ def add_or_match_org(
         org_type,
     ):
         key = (
-            clean_text(acronym),
-            clean_text(org_name),
+            normalise(acronym),
+            normalise(org_name),
         )
         if key in self.data:
             org_type_old = self.data[key][2]
@@ -68,8 +68,8 @@ def add_or_match_org(
             return
         self.data[
             (
-                clean_text(acronym),
-                clean_text(org_name),
+                normalise(acronym),
+                normalise(org_name),
             )
         ] = [acronym, org_name, org_type]
 
@@ -93,9 +93,9 @@ def get_org_info(self, org_name: str, location: str) -> Dict[str, str]:
         org_map_info = org_name_map.get(org_name)
         if not org_map_info:
             org_name_map_clean = {
-                clean_text(on): org_name_map[on] for on in org_name_map
+                normalise(on): org_name_map[on] for on in org_name_map
             }
-            org_name_clean = clean_text(org_name)
+            org_name_clean = normalise(org_name)
             org_map_info = org_name_map_clean.get(org_name_clean)
         if not org_map_info:
             return {"#org+name": org_name}

src/hapi/pipelines/utilities/mappings.py

@@ -1,8 +1,7 @@
-from re import sub
 from typing import Dict
 
-from hdx.location.names import clean_name
 from hdx.location.phonetics import Phonetics
+from hdx.utilities.text import normalise
 
 MATCH_THRESHOLD = 5
 
@@ -30,8 +29,8 @@ def get_code_from_name(
     code = code_lookup.get(name)
     if code:
         return code, name, False
-    name_clean = clean_text(name)
-    clean_lookup = {clean_text(c): code_lookup[c] for c in code_lookup}
+    name_clean = normalise(name)
+    clean_lookup = {normalise(c): code_lookup[c] for c in code_lookup}
     code = clean_lookup.get(name_clean)
     if code:
         return code, name_clean, False
@@ -54,10 +53,3 @@ def get_code_from_name(
     name = names[name_index]
     code = code_lookup.get(name, code_mapping.get(name))
     return code, name_clean, True
-
-
-def clean_text(text: str) -> str:
-    text_clean = clean_name(text)
-    text_clean = sub(r"[^'a-zA-Z0-9\s]", " ", text_clean)
-    text_clean = sub(" +", " ", text_clean)
-    return text_clean.strip()