Adds check for performance DB (#56)

* adds check_performance

* adds rule for check

* adds check rule

* refactors error message

* change print statements to logging

* add conn.close()

* adds finally block

Author: CarlosCoelhoSL

.github/workflows/run.yml

@@ -49,6 +49,9 @@ jobs:
     - name: Build dataset
       run: make
 
+    - name: Check performance database
+      run: make check-performance
+
     # Development
     - name: Configure Development AWS Credentials
       uses: aws-actions/configure-aws-credentials@v1-node16

.github/workflows/run_dev.yml

@@ -47,6 +47,9 @@ jobs:
     - name: Build dataset
       run: make
 
+    - name: Check performance database
+      run: make check-performance
+
     # Development
     - name: Configure Development AWS Credentials
       uses: aws-actions/configure-aws-credentials@v1-node16

.github/workflows/run_performance.yml

@@ -49,6 +49,9 @@ jobs:
     - name: Build performance dataset
       run: make third-pass
 
+    - name: Check performance database
+      run: make check-performance
+
     # Development
     - name: Configure Development AWS Credentials
       uses: aws-actions/configure-aws-credentials@v1-node16

.github/workflows/run_performance_dev.yml

@@ -49,6 +49,9 @@ jobs:
     - name: Build performance dataset
       run: make third-pass
 
+    - name: Check performance database
+      run: make check-performance
+
     # Development
     - name: Configure Development AWS Credentials
       uses: aws-actions/configure-aws-credentials@v1-node16

Makefile

@@ -97,3 +97,6 @@ specification::
 	curl -qfsL '$(SOURCE_URL)/specification/main/specification/provision.csv' > specification/provision.csv
 	curl -qfsL '$(SOURCE_URL)/specification/main/specification/provision-rule.csv' > specification/provision-rule.csv
 	curl -qfsL '$(SOURCE_URL)/specification/main/specification/provision-reason.csv' > specification/provision-reason.csv
+
+check-performance::
+	python check/performance.py

check/performance.py

@@ -0,0 +1,35 @@
+import logging
+import sqlite3
+from digital_land.expectations.operation import (
+    check_columns
+)
+
+logger = logging.getLogger("__name__")
+
+EXPECTED = {
+    "endpoint_dataset_issue_type_summary": ['organisation', 'organisation_name', 'cohort', 'dataset', 'collection', 'pipeline', 'endpoint', 'endpoint_url', 'resource', 'resource_start_date', 'resource_end_date', 'latest_log_entry_date', 'count_issues', 'date', 'issue_type', 'severity', 'responsibility', 'fields'],
+    "endpoint_dataset_resource_summary": ['organisation', 'organisation_name', 'cohort', 'dataset', 'collection', 'pipeline', 'endpoint', 'endpoint_url', 'resource', 'resource_start_date', 'resource_end_date', 'latest_log_entry_date', 'mapping_field', 'non_mapping_field'],
+    "endpoint_dataset_summary": ['organisation', 'dataset', 'endpoint', 'endpoint_url', 'resource', 'latest_status', 'latest_exception', 'latest_log_entry_date', 'entry_date', 'end_date', 'latest_resource_start_date', 'resource_end_date'],
+    "provision_summary": ['organisation', 'organisation_name', 'dataset', 'provision_reason', 'active_endpoint_count', 'error_endpoint_count', 'count_issue_error_internal', 'count_issue_error_external', 'count_issue_warning_internal', 'count_issue_warning_external', 'count_issue_notice_internal', 'count_issue_notice_external'],
+    "reporting_historic_endpoints": ['organisation', 'name', 'organisation_name', 'dataset', 'collection', 'pipeline', 'endpoint', 'endpoint_url', 'licence', 'latest_status', 'latest_exception', 'resource', 'latest_log_entry_date', 'endpoint_entry_date', 'endpoint_end_date', 'resource_start_date', 'resource_end_date'],
+    "reporting_latest_endpoints": ['organisation', 'name', 'organisation_name', 'dataset', 'collection', 'pipeline', 'endpoint', 'endpoint_url', 'licence', 'latest_status', 'days_since_200', 'latest_exception', 'resource', 'latest_log_entry_date', 'endpoint_entry_date', 'endpoint_end_date', 'resource_start_date', 'resource_end_date', 'rn']
+}
+
+def check_performance_columns():
+    try:
+        conn = sqlite3.connect("dataset/performance.sqlite3").cursor()
+        result, message, details = check_columns(conn, EXPECTED)
+        if not result:
+            logging.error("Column check failed for performance DB")
+            logging.error(message)
+            for item in details:
+                if not item["success"]:
+                    logging.error(f"{item['table']} did not have all expected columns. Missing columns: {item['missing']}")
+                    logging.error(f"Columns found: {item['actual']}")
+            raise Exception(f"Performance DB check failed: {message}")
+    finally:
+        conn.close()
+
+
+if __name__ == "__main__":
+    check_performance_columns()