Update HAPI pipelines to convert xlsx to csv before processing for conflict events (OCHA-DAP#140)

* Update HAPI pipelines to convert xlsx to csv before processing for conflict events

* Update CHANGELOG

Author: mcarans

.config/pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
       - id: end-of-file-fixer
       - id: check-ast
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.1
+    rev: v0.5.5
     hooks:
       # Run the linter.
       - id: ruff
@@ -17,7 +17,7 @@ repos:
       - id: ruff-format
         args: [--config, .config/ruff.toml]
   - repo: https://github.com/astral-sh/uv-pre-commit
-    rev: 0.2.13
+    rev: 0.2.31
     hooks:
       # Run the pip compile
       - id: pip-compile

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [0.9.41] - 2024-07-31
+
+### Changed
+
+- Convert xlsx to csv for conflict events
+
 ## [0.9.40] - 2024-07-30
 
 ### Changed

pyproject.toml

@@ -38,8 +38,8 @@ dependencies = [
     "hdx-python-api>= 6.3.2",
     "hdx-python-country>= 3.7.7",
     "hdx-python-database[postgresql]>= 1.3.1",
-    "hdx-python-scraper>= 2.4.0",
-    "hdx-python-utilities>= 3.7.2",
+    "hdx-python-scraper>= 2.4.1",
+    "hdx-python-utilities>= 3.7.3",
     "libhxl",
     "sqlalchemy"
 ]

requirements.txt

@@ -71,9 +71,9 @@ hdx-python-country==3.7.7
     #   hdx-python-scraper
 hdx-python-database==1.3.1
     # via hapi-pipelines (pyproject.toml)
-hdx-python-scraper==2.4.0
+hdx-python-scraper==2.4.1
     # via hapi-pipelines (pyproject.toml)
-hdx-python-utilities==3.7.2
+hdx-python-utilities==3.7.3
     # via
     #   hapi-pipelines (pyproject.toml)
     #   hdx-python-api
@@ -305,6 +305,8 @@ xlrd==2.0.1
     # via hdx-python-utilities
 xlrd3==1.1.0
     # via libhxl
+xlsx2csv==0.8.3
+    # via hdx-python-utilities
 xlsxwriter==3.2.0
     # via tableschema-to-template
 xlwt==1.3.0

src/hapi/pipelines/configs/conflict_event.yaml

@@ -81,6 +81,7 @@ conflict_event_default:
     - "conflict_event_yem_civilian_targeting"
     - "conflict_event_yem_demonstration"
   format: "xlsx"
+  xlsx2csv: True
   sheet: "Data"
   admin:
     - "ISO3"

src/hapi/pipelines/utilities/process_config_defaults.py

@@ -56,6 +56,7 @@ def _combine_default(country: Dict, default: Dict) -> Dict:
             country[list_name] = country[list_name] + default[list_name]
     for other_parameter in (
         "format",
+        "xlsx2csv",
         "sheet",
         "headers",
         "use_hxl",