Skip to content

Commit 23fa283

Browse files
alexandru-m-galexandru-m-g
authored
Merge pull request #17 from OCHA-DAP/main
HDXDSYS-924 COD and TCD 3W
2 parents db84674 + c01734a commit 23fa283

11 files changed

+344
-312
lines changed

‎.config/pre-commit-config.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ repos:
88
- id: end-of-file-fixer
99
- id: check-ast
1010
- repo: https://github.com/astral-sh/ruff-pre-commit
11-
rev: v0.5.1
11+
rev: v0.5.5
1212
hooks:
1313
# Run the linter.
1414
- id: ruff
@@ -17,7 +17,7 @@ repos:
1717
- id: ruff-format
1818
args: [--config, .config/ruff.toml]
1919
- repo: https://github.com/astral-sh/uv-pre-commit
20-
rev: 0.2.13
20+
rev: 0.2.31
2121
hooks:
2222
# Run the pip compile
2323
- id: pip-compile

‎CHANGELOG.md

+18
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,24 @@ All notable changes to this project will be documented in this file.
44

55
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
66

7+
## [0.9.41] - 2024-07-31
8+
9+
### Changed
10+
11+
- Convert xlsx to csv for conflict events
12+
13+
## [0.9.40] - 2024-07-30
14+
15+
### Changed
16+
17+
- Split pipelines test into multiple tests
18+
19+
## [0.9.39] - 2024-07-29
20+
21+
### Fixed
22+
23+
- COD and TCD operational presence update
24+
725
## [0.9.38] - 2024-07-26
826

927
### Fixed

‎pyproject.toml

+3-3
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,11 @@ requires-python = ">=3.8"
3535

3636
dependencies = [
3737
"hapi-schema>=0.8.14",
38-
"hdx-python-api>= 6.3.1",
38+
"hdx-python-api>= 6.3.2",
3939
"hdx-python-country>= 3.7.7",
4040
"hdx-python-database[postgresql]>= 1.3.1",
41-
"hdx-python-scraper>= 2.4.0",
42-
"hdx-python-utilities>= 3.7.2",
41+
"hdx-python-scraper>= 2.4.1",
42+
"hdx-python-utilities>= 3.7.3",
4343
"libhxl",
4444
"sqlalchemy"
4545
]

‎requirements.txt

+10-8
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ gspread==6.1.2
6060
# via hdx-python-scraper
6161
hapi-schema==0.8.14
6262
# via hapi-pipelines (pyproject.toml)
63-
hdx-python-api==6.3.1
63+
hdx-python-api==6.3.2
6464
# via
6565
# hapi-pipelines (pyproject.toml)
6666
# hdx-python-scraper
@@ -71,9 +71,9 @@ hdx-python-country==3.7.7
7171
# hdx-python-scraper
7272
hdx-python-database==1.3.1
7373
# via hapi-pipelines (pyproject.toml)
74-
hdx-python-scraper==2.4.0
74+
hdx-python-scraper==2.4.1
7575
# via hapi-pipelines (pyproject.toml)
76-
hdx-python-utilities==3.7.2
76+
hdx-python-utilities==3.7.3
7777
# via
7878
# hapi-pipelines (pyproject.toml)
7979
# hdx-python-api
@@ -150,7 +150,7 @@ ply==3.11
150150
# libhxl
151151
pockets==0.9.1
152152
# via sphinxcontrib-napoleon
153-
pre-commit==3.7.1
153+
pre-commit==3.8.0
154154
# via hapi-pipelines (pyproject.toml)
155155
psycopg==3.2.1
156156
# via hdx-python-database
@@ -178,7 +178,7 @@ pyopenssl==24.2.1
178178
# ndg-httpsclient
179179
pyphonetics==0.5.3
180180
# via hdx-python-country
181-
pytest==8.3.1
181+
pytest==8.3.2
182182
# via
183183
# hapi-pipelines (pyproject.toml)
184184
# pytest-check
@@ -214,7 +214,7 @@ referencing==0.35.1
214214
# via
215215
# jsonschema
216216
# jsonschema-specifications
217-
regex==2024.5.15
217+
regex==2024.7.24
218218
# via hdx-python-scraper
219219
requests==2.32.3
220220
# via
@@ -232,7 +232,7 @@ rfc3986==2.0.0
232232
# via frictionless
233233
rich==13.7.1
234234
# via typer
235-
rpds-py==0.19.0
235+
rpds-py==0.19.1
236236
# via
237237
# jsonschema
238238
# referencing
@@ -242,7 +242,7 @@ ruamel-yaml==0.18.6
242242
# via hdx-python-utilities
243243
ruamel-yaml-clib==0.2.8
244244
# via ruamel-yaml
245-
setuptools==71.1.0
245+
setuptools==72.1.0
246246
# via ckanapi
247247
shellingham==1.5.4
248248
# via typer
@@ -305,6 +305,8 @@ xlrd==2.0.1
305305
# via hdx-python-utilities
306306
xlrd3==1.1.0
307307
# via libhxl
308+
xlsx2csv==0.8.3
309+
# via hdx-python-utilities
308310
xlsxwriter==3.2.0
309311
# via tableschema-to-template
310312
xlwt==1.3.0

‎src/hapi/pipelines/configs/conflict_event.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ conflict_event_default:
8181
- "conflict_event_yem_civilian_targeting"
8282
- "conflict_event_yem_demonstration"
8383
format: "xlsx"
84+
xlsx2csv: True
8485
sheet: "Data"
8586
admin:
8687
- "ISO3"

‎src/hapi/pipelines/configs/operational_presence.yaml

+4-4
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ operational_presence_admintwo:
102102

103103
operational_presence_cod:
104104
dataset: "drc_presence_operationnelle"
105-
resource: "rdc-3w_avril-2024.xlsx"
105+
resource: "rdc-3w_avril-2024"
106106
format: "xlsx"
107107
sheet: "Compilation_VF"
108108
headers: 1
@@ -384,13 +384,13 @@ operational_presence_admintwo:
384384
operational_presence_adminone:
385385
operational_presence_tcd:
386386
dataset: "chad-operational-presence"
387-
resource: "3W_TCD_May2024"
387+
resource: "3W_TCD_June2024"
388388
format: "xlsx"
389389
sheet: "3w opérationnelle"
390390
headers: 1
391391
source_date:
392-
start: "01/12/2023"
393-
end: "31/05/2024"
392+
start: "01/06/2024"
393+
end: "30/06/2024"
394394
use_hxl: False
395395
admin:
396396
- ~

‎src/hapi/pipelines/utilities/process_config_defaults.py

+1
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ def _combine_default(country: Dict, default: Dict) -> Dict:
5656
country[list_name] = country[list_name] + default[list_name]
5757
for other_parameter in (
5858
"format",
59+
"xlsx2csv",
5960
"sheet",
6061
"headers",
6162
"use_hxl",

‎tests/__init__.py

Whitespace-only changes.

‎tests/org_mappings.py

+172
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
from hapi.pipelines.database.org import OrgInfo
2+
3+
4+
def check_org_mappings(pipelines):
5+
org_map = pipelines.org._org_map
6+
iom = OrgInfo(
7+
"International Organization for Migration",
8+
"international organization for migration",
9+
"IOM",
10+
"iom",
11+
"447",
12+
True,
13+
True,
14+
)
15+
assert org_map[(None, "IOM")] == iom
16+
assert org_map[(None, "iom")] == iom
17+
assert (
18+
org_map[
19+
(
20+
None,
21+
"Organisation Internationale pour les Migrations",
22+
)
23+
]
24+
== iom
25+
)
26+
assert (
27+
org_map[
28+
(
29+
None,
30+
"organisation internationale pour les migrations",
31+
)
32+
]
33+
== iom
34+
)
35+
36+
iom = OrgInfo(
37+
"International Organization for Migration",
38+
"international organization for migration",
39+
"IOM",
40+
"iom",
41+
"447",
42+
False,
43+
False,
44+
)
45+
assert org_map[(None, "International Organisation for Migrations")] == iom
46+
assert org_map[(None, "international organisation for migrations")] == iom
47+
assert org_map[(None, "INTERNATIONALE ORGANISATION FOR MIGRATION")] == iom
48+
assert org_map[(None, "internationale organisation for migration")] == iom
49+
assert (
50+
org_map[
51+
(
52+
None,
53+
"Organisation Internationale des Migrations",
54+
)
55+
]
56+
== iom
57+
)
58+
assert (
59+
org_map[
60+
(
61+
None,
62+
"organisation internationale des migrations",
63+
)
64+
]
65+
== iom
66+
)
67+
assert (
68+
org_map[
69+
(
70+
None,
71+
"OIM - International Organization for Migration",
72+
)
73+
]
74+
== iom
75+
)
76+
assert (
77+
org_map[
78+
(
79+
None,
80+
"oim international organization for migration",
81+
)
82+
]
83+
== iom
84+
)
85+
86+
unicef = OrgInfo(
87+
"United Nations Children's Fund",
88+
"united nations childrens fund",
89+
"UNICEF",
90+
"unicef",
91+
"447",
92+
True,
93+
True,
94+
)
95+
assert org_map[(None, "United Nations Children's Fund")] == unicef
96+
assert org_map[(None, "united nations childrens fund")] == unicef
97+
assert org_map[(None, "UNICEF")] == unicef
98+
assert org_map[(None, "unicef")] == unicef
99+
assert org_map[(None, "Fonds des Nations Unies pour l'Enfance")] == unicef
100+
assert org_map[(None, "fonds des nations unies pour lenfance")] == unicef
101+
assert (
102+
org_map[
103+
(
104+
None,
105+
"UNICEF - Fondo de las Naciones Unidas para la Infancia",
106+
)
107+
]
108+
== unicef
109+
)
110+
assert (
111+
org_map[
112+
(
113+
None,
114+
"unicef fondo de las naciones unidas para la infancia",
115+
)
116+
]
117+
== unicef
118+
)
119+
120+
unicef = OrgInfo(
121+
"United Nations Children's Fund",
122+
"united nations childrens fund",
123+
"UNICEF",
124+
"unicef",
125+
"447",
126+
False,
127+
False,
128+
)
129+
assert (
130+
org_map[(None, "United Nations Children's Emergency Fund")] == unicef
131+
)
132+
assert org_map[(None, "united nations childrens emergency fund")] == unicef
133+
assert org_map[(None, "Fond des Nations Unies pour l'Enfance")] == unicef
134+
assert org_map[(None, "fond des nations unies pour lenfance")] == unicef
135+
assert (
136+
org_map[
137+
(
138+
None,
139+
"United Nations International Childrens Emergency Fund",
140+
)
141+
]
142+
== unicef
143+
)
144+
assert (
145+
org_map[
146+
(
147+
None,
148+
"united nations international childrens emergency fund",
149+
)
150+
]
151+
== unicef
152+
)
153+
154+
assert org_map[("AFG", "WEWORLD")] == OrgInfo(
155+
"WEWORLD",
156+
"weworld",
157+
"WEWORLD",
158+
"weworld",
159+
None,
160+
True,
161+
False,
162+
)
163+
164+
assert org_map[("NGA", "HECADF")] == OrgInfo(
165+
"HECADF",
166+
"hecadf",
167+
"HECADF",
168+
"hecadf",
169+
"441",
170+
True,
171+
True,
172+
)

0 commit comments

Comments
 (0)