switch to endpoint dataset summary table (#49)

Author: CarlosCoelhoSL

application/data_access/odp_summaries/conformance.py

@@ -55,8 +55,12 @@ def get_column_field_summary(dataset_clause, offset):
         SELECT endpoint, licence
         FROM reporting_latest_endpoints
     ) AS rle ON edrs.endpoint = rle.endpoint
+    LEFT JOIN (
+        SELECT endpoint, end_date as endpoint_end_date
+        FROM endpoint_dataset_summary
+    ) as eds on edrs.endpoint = eds.endpoint
     WHERE edrs.resource != ''
-    and edrs.endpoint_end_date=''
+    and eds.endpoint_end_date=''
     and ({dataset_clause})
     limit 1000 offset {offset}
     """
@@ -67,7 +71,7 @@ def get_column_field_summary(dataset_clause, offset):
 
 def get_issue_summary(dataset_clause, offset):
     sql = f"""
-    select  * from endpoint_dataset_issue_type_summary
+    select  * from endpoint_dataset_issue_type_summary edrs
     where ({dataset_clause})
     limit 1000 offset {offset}
     """
@@ -89,7 +93,7 @@ def get_odp_conformance_summary(dataset_types, cohorts):
     else:
         datasets = ALL_DATASETS
     dataset_clause = " or ".join(
-        ("pipeline = '" + str(dataset) + "'" for dataset in datasets)
+        ("edrs.pipeline = '" + str(dataset) + "'" for dataset in datasets)
     )
 
     provision_df = get_provisions(cohorts, COHORTS)

application/data_access/odp_summaries/issue.py

@@ -280,9 +280,14 @@ def get_issue_summary_by_issue_type(dataset_clause, offset):
     SELECT
         *
     FROM
-        endpoint_dataset_issue_type_summary
+        endpoint_dataset_issue_type_summary edits
+    LEFT JOIN (
+        SELECT endpoint, end_date as endpoint_end_date, latest_status,
+                latest_exception, entry_date as endpoint_entry_date
+        FROM endpoint_dataset_summary
+    ) as eds on edits.endpoint = eds.endpoint
     {dataset_clause}
-    and endpoint_end_date = ''
+    and eds.endpoint_end_date = ''
     limit 1000 offset {offset}
     """
     return get_datasette_query("performance", sql)
@@ -299,7 +304,7 @@ def get_odp_issues_by_issue_type(dataset_types, cohorts):
         datasets = ALL_DATASETS
 
     dataset_clause = "WHERE " + " or ".join(
-        ("dataset = '" + str(dataset) + "'" for dataset in datasets)
+        ("edits.dataset = '" + str(dataset) + "'" for dataset in datasets)
     )
 
     pagination_incomplete = True
@@ -329,8 +334,8 @@ def get_odp_issues_by_issue_type(dataset_types, cohorts):
             "collection",
             "endpoint",
             "endpoint_url",
-            "status",
-            "exception",
+            "latest_status",
+            "latest_exception",
             "resource",
             "latest_log_entry_date",
             "endpoint_entry_date",