Merge pull request #18 from OCHA-DAP/main

HDXDSYS-941 fuzzy matching and SDN population

Author: alexandru-m-g

CHANGELOG.md

@@ -4,6 +4,18 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [0.9.43] - 2024-08-01
+
+### Changed
+
+- Updated SDN population data
+
+## [0.9.42] - 2024-08-01
+
+### Changed
+
+- Reenable fuzzy matching
+
 ## [0.9.41] - 2024-07-31
 
 ### Changed

src/hapi/pipelines/configs/population.yaml

@@ -840,7 +840,7 @@ population_national:
 
   population_sdn:
     dataset: "cod-ps-sdn"
-    resource: "sdn_admpop_adm0_2022.csv"
+    resource: "sdn_admpop_adm0_2024.csv"
     format: "csv"
     use_hxl: False
     admin_single: "SDN"
@@ -1819,7 +1819,7 @@ population_adminone:
 
   population_sdn:
     dataset: "cod-ps-sdn"
-    resource: "sdn_admpop_adm1_2022.csv"
+    resource: "sdn_admpop_adm1_2024.csv"
     format: "csv"
     use_hxl: False
     admin:
@@ -2747,6 +2747,27 @@ population_admintwo:
       - "#population+m+age_80_plus"
       - "#population+age_80_plus+total"
 
+  population_sdn:
+    dataset: "cod-ps-sdn"
+    resource: "sdn_admpop_adm2_2024.csv"
+    format: "csv"
+    use_hxl: False
+    admin:
+      - ~
+      - "ADM2_PCODE"
+    input:
+      - "F_80Plus"
+      - "M_80Plus"
+      - "T_80Plus"
+    output:
+      - "F_80plus"
+      - "M_80plus"
+      - "T_80plus"
+    output_hxl:
+      - "#population+f+age_80_plus"
+      - "#population+m+age_80_plus"
+      - "#population+age_80_plus+total"
+
   population_slv:
     dataset: "cod-ps-slv"
     resource: "slv_admpop_adm2_2023.csv"

src/hapi/pipelines/database/org_type.py

@@ -24,6 +24,7 @@ def __init__(
         super().__init__(session)
         self._datasetinfo = datasetinfo
         self.data = org_type_map
+        self.unmatched = []
 
     def populate(self):
         logger.info("Populating org type table")
@@ -67,4 +68,5 @@ def get_org_type_code(self, org_type: str) -> str | None:
         return get_code_from_name(
             name=org_type,
             code_lookup=self.data,
+            unmatched=self.unmatched,
         )

src/hapi/pipelines/database/sector.py

@@ -24,6 +24,7 @@ def __init__(
         super().__init__(session)
         self._datasetinfo = datasetinfo
         self.data = sector_map
+        self.unmatched = []
 
     def populate(self):
         logger.info("Populating sector table")
@@ -64,4 +65,5 @@ def get_sector_code(self, sector: str) -> str | None:
         return get_code_from_name(
             name=sector,
             code_lookup=self.data,
+            unmatched=self.unmatched,
         )

src/hapi/pipelines/utilities/mappings.py

@@ -1,4 +1,4 @@
-from typing import Dict
+from typing import Dict, List
 
 from hdx.location.phonetics import Phonetics
 from hdx.utilities.text import normalise
@@ -9,14 +9,16 @@
 def get_code_from_name(
     name: str,
     code_lookup: Dict[str, str],
-    fuzzy_match: bool = False,
+    unmatched: List[str],
+    fuzzy_match: bool = True,
 ) -> str | None:
     """
     Given a name (org type, sector, etc), return the corresponding code.
 
     Args:
         name (str): Name to match
         code_lookup (dict): Dictionary of official names and codes
+        unmatched (List[str]): List of unmatched names
         fuzzy_match (bool): Allow fuzzy matching or not
 
     Returns:
@@ -25,22 +27,27 @@ def get_code_from_name(
     code = code_lookup.get(name)
     if code:
         return code
+    if name in unmatched:
+        return None
     name_clean = normalise(name)
     code = code_lookup.get(name_clean)
     if code:
         code_lookup[name] = code
         return code
     if len(name) <= MATCH_THRESHOLD:
+        unmatched.append(name)
         return None
     if not fuzzy_match:
+        unmatched.append(name)
         return None
-    names = list(code_lookup.keys())
+    names = [x for x in code_lookup.keys() if len(x) > MATCH_THRESHOLD]
     name_index = Phonetics().match(
         possible_names=names,
         name=name,
         alternative_name=name_clean,
     )
     if name_index is None:
+        unmatched.append(name)
         return None
     code = code_lookup.get(names[name_index])
     if code:

tests/test_mappings.py

@@ -38,21 +38,51 @@ def test_get_code_from_name_org_type():
         get_code_from_name(
             "NATIONAL_NGO",
             actual_org_type_lookup,
+            [],
+            fuzzy_match=False,
         )
         == "441"
     )
     assert (
         get_code_from_name(
             "COOPÉRATION_INTERNATIONALE",
             actual_org_type_lookup,
+            [],
+            fuzzy_match=False,
+        )
+        is None
+    )
+    unmatched = []
+    assert (
+        get_code_from_name(
+            "COOPÉRATION_INTERNATIONALE",
+            actual_org_type_lookup,
+            unmatched,
+            fuzzy_match=True,
+        )
+        is None
+    )
+    assert (
+        get_code_from_name(
+            "COOPÉRATION_INTERNATIONALE",
+            actual_org_type_lookup,
+            unmatched,
+            fuzzy_match=True,
+        )
+        is None
+    )
+    assert (
+        get_code_from_name(
+            "NGO", actual_org_type_lookup, [], fuzzy_match=False
         )
         is None
     )
-    assert get_code_from_name("NGO", actual_org_type_lookup) is None
     assert (
         get_code_from_name(
             "International",
             actual_org_type_lookup,
+            [],
+            fuzzy_match=False,
         )
         is None
     )
@@ -100,16 +130,29 @@ def test_get_code_from_name_sector():
     actual_sector_lookup = {normalise(k): v for k, v in sector_lookup.items()}
     actual_sector_lookup.update(sector_map)
     assert (
-        get_code_from_name("education", actual_sector_lookup, fuzzy_match=True)
+        get_code_from_name(
+            "education", actual_sector_lookup, [], fuzzy_match=True
+        )
         == "EDU"
     )
     assert (
         get_code_from_name(
-            "LOGISTIQUE", actual_sector_lookup, fuzzy_match=True
+            "LOGISTIQUE", actual_sector_lookup, [], fuzzy_match=True
         )
         == "LOG"
     )
-    assert get_code_from_name("CCCM", actual_sector_lookup) == "CCM"
-    assert get_code_from_name("Santé", actual_sector_lookup) == "HEA"
+    assert (
+        get_code_from_name("CCCM", actual_sector_lookup, [], fuzzy_match=False)
+        == "CCM"
+    )
+    assert (
+        get_code_from_name(
+            "Santé", actual_sector_lookup, [], fuzzy_match=False
+        )
+        == "HEA"
+    )
     actual_sector_lookup["cccm"] = "CCM"
-    assert get_code_from_name("CCS", actual_sector_lookup) is None
+    assert (
+        get_code_from_name("CCS", actual_sector_lookup, [], fuzzy_match=False)
+        is None
+    )