From 8d69668553b48a5293dfe449980c60f3a3748b9b Mon Sep 17 00:00:00 2001 From: CarlosCoelhoSL Date: Tue, 18 Feb 2025 16:20:48 +0000 Subject: [PATCH 1/5] adds operation and test --- digital_land/expectations/operation.py | 35 +++++++++++ .../expectations/test_operation.py | 61 +++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/digital_land/expectations/operation.py b/digital_land/expectations/operation.py index 453d9a38..d67216f2 100644 --- a/digital_land/expectations/operation.py +++ b/digital_land/expectations/operation.py @@ -186,3 +186,38 @@ def count_deleted_entities( } return result, message, details + + +def check_columns(conn, expected: dict): + details = [] + success_count = 0 + failure_count = 0 + # expected: a dictionary containing table names as keys, with their a list of their expected columns as the value + for k, v in expected.items(): + table_name = k + expected_columns = v + sql = f""" + PRAGMA table_info({table_name}) + """ + rows = conn.execute(sql).fetchall() + actual = [row[1] for row in rows] + success = set(expected_columns).issubset(set(actual)) + missing = list(set(expected_columns) - set(actual)) + details.append( + { + "table": table_name, + "success": success, + "missing": missing, + "actual": actual, + "expected": expected_columns, + } + ) + if success: + success_count += 1 + else: + failure_count += 1 + + result = False if failure_count > 0 else True + message = f"{success_count} out of {success_count + failure_count} tables had expected columns" + + return result, message, details diff --git a/tests/integration/expectations/test_operation.py b/tests/integration/expectations/test_operation.py index 1efe87d9..5abdc307 100644 --- a/tests/integration/expectations/test_operation.py +++ b/tests/integration/expectations/test_operation.py @@ -1,8 +1,10 @@ import spatialite +import sqlite3 import pytest import pandas as pd from digital_land.expectations.operation import ( + check_columns, count_lpa_boundary, count_deleted_entities, ) @@ -174,3 +176,62 @@ def test_count_deleted_entities(dataset_path, mocker): for key in detail_keys: assert key in details, f"{key} missing from details" assert "1002" in details["entities"] + + +def test_check_columns(dataset_path): + expected = { + "entity": [ + "dataset", + "end_date", + "entity", + "entry_date", + "geojson", + "geometry", + "json", + "name", + "organisation_entity", + "point", + "prefix", + "reference", + "start_date", + "typology", + ], + "old_entity": ["old_entity", "entity"], + } + + with sqlite3.connect(dataset_path) as conn: + result, message, details = check_columns(conn.cursor(), expected) + + assert result + assert "2 out of 2 tables had expected columns" in message + + +def test_check_columns_failure(dataset_path): + expected = { + "entity": [ + "missing", + "columns", + "dataset", + "end_date", + "entity", + "entry_date", + "geojson", + "geometry", + "json", + "name", + "organisation_entity", + "point", + "prefix", + "reference", + "start_date", + "typology", + ], + "old_entity": ["old_entity", "entity"], + } + + with sqlite3.connect(dataset_path) as conn: + result, message, details = check_columns(conn.cursor(), expected) + assert result + assert "1 out of 2 tables had expected columns" in message + assert not details[0]["success"] + assert details[0]["missing"] == ["missing", "columns"] From 9f1813b899600c8d4c9e7eda05b35a28d32b37a6 Mon Sep 17 00:00:00 2001 From: CarlosCoelhoSL Date: Tue, 18 Feb 2025 16:25:36 +0000 Subject: [PATCH 2/5] fixes test --- tests/integration/expectations/test_operation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/expectations/test_operation.py b/tests/integration/expectations/test_operation.py index 5abdc307..2a17e268 100644 --- a/tests/integration/expectations/test_operation.py +++ b/tests/integration/expectations/test_operation.py @@ -231,7 +231,7 @@ def test_check_columns_failure(dataset_path): with sqlite3.connect(dataset_path) as conn: result, message, details = check_columns(conn.cursor(), expected) - assert result + assert not result assert "1 out of 2 tables had expected columns" in message assert not details[0]["success"] assert details[0]["missing"] == ["missing", "columns"] From a5d093415d1422776d6cf7da3be709330d97b436 Mon Sep 17 00:00:00 2001 From: CarlosCoelhoSL Date: Tue, 18 Feb 2025 16:30:16 +0000 Subject: [PATCH 3/5] fixes test (again) --- tests/integration/expectations/test_operation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/expectations/test_operation.py b/tests/integration/expectations/test_operation.py index 2a17e268..91279faf 100644 --- a/tests/integration/expectations/test_operation.py +++ b/tests/integration/expectations/test_operation.py @@ -234,4 +234,5 @@ def test_check_columns_failure(dataset_path): assert not result assert "1 out of 2 tables had expected columns" in message assert not details[0]["success"] - assert details[0]["missing"] == ["missing", "columns"] + assert "missing" in details[0]["missing"] + assert "columns" in details[0]["missing"] From cf6bcbcb108f728c73cd4d9976a943c2983f0e37 Mon Sep 17 00:00:00 2001 From: CarlosCoelhoSL Date: Tue, 18 Feb 2025 16:34:07 +0000 Subject: [PATCH 4/5] adds comments --- digital_land/expectations/operation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/digital_land/expectations/operation.py b/digital_land/expectations/operation.py index d67216f2..934b05ef 100644 --- a/digital_land/expectations/operation.py +++ b/digital_land/expectations/operation.py @@ -189,10 +189,13 @@ def count_deleted_entities( def check_columns(conn, expected: dict): + # This operation checks that the db connection provided contains the tables with the expected columns provided + + # expected: a dictionary containing table names as keys, with a list of their expected columns as the value + details = [] success_count = 0 failure_count = 0 - # expected: a dictionary containing table names as keys, with their a list of their expected columns as the value for k, v in expected.items(): table_name = k expected_columns = v From c9756d9bec78f9f21d03910be89393357dff1147 Mon Sep 17 00:00:00 2001 From: CarlosCoelhoSL Date: Tue, 18 Feb 2025 16:49:07 +0000 Subject: [PATCH 5/5] expands tests --- tests/integration/expectations/test_operation.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/expectations/test_operation.py b/tests/integration/expectations/test_operation.py index 91279faf..3fc72d12 100644 --- a/tests/integration/expectations/test_operation.py +++ b/tests/integration/expectations/test_operation.py @@ -205,6 +205,10 @@ def test_check_columns(dataset_path): assert result assert "2 out of 2 tables had expected columns" in message + assert details[0]["table"] == "entity" + assert any(x in details[0]["actual"] for x in expected["entity"]) + assert any(x in details[0]["expected"] for x in expected["entity"]) + def test_check_columns_failure(dataset_path): expected = {