Merge pull request #48 from digital-land/overview_report_modificxation

Update Issue Summary Table

Author: averheecke-tpx

application/blueprints/report/views.py

@@ -36,8 +36,8 @@
     get_grouped_entity_count,
 )
 from application.data_access.overview.issue_summary import (
-    get_issue_summary,
-    get_issue_summary_for_csv,
+    get_full_issue_summary,
+    get_full_issue_summary_for_csv,
 )
 from application.data_access.overview.source_and_resource_queries import (
     get_datasets_summary,
@@ -96,7 +96,7 @@ def overview():
         "internal_errors_timeseries": internal_errors_timeseries,
     }
 
-    issue_summary = get_issue_summary()
+    issue_summary = get_full_issue_summary()
 
     return render_template(
         "reporting/overview.html",
@@ -162,8 +162,8 @@ def download_csv():
         )
         file_path = generate_odp_summary_csv(conformance_df)
         return send_file(file_path, download_name="odp-conformance.csv")
-    if type == "issue-summary":
-        overview_issue_summary = get_issue_summary_for_csv()
+    if type == "endpoint_dataset_issue_type_summary":
+        overview_issue_summary = get_full_issue_summary_for_csv()
         file_path = generate_overview_issue_summary_csv(overview_issue_summary)
         return send_file(file_path, download_name="overview_issue_summary.csv")
 

application/data_access/overview/issue_summary.py

@@ -1,10 +1,18 @@
-from application.data_access.datasette_utils import get_datasette_query_issue_summary
+import pandas as pd
 
+from application.data_access.datasette_utils import get_datasette_query
 
-def get_issue_summary():
-    issues_df = get_datasette_query_issue_summary(
-        "performance/endpoint_dataset_issue_type_summary"
-    )
+
+def get_full_issue_summary():
+    pagination_incomplete = True
+    offset = 0
+    issue_summary_df_list = []
+    while pagination_incomplete:
+        issue_summary_df = get_issue_summary(offset)
+        issue_summary_df_list.append(issue_summary_df)
+        pagination_incomplete = len(issue_summary_df) == 1000
+        offset += 1000
+    issues_df = pd.concat(issue_summary_df_list)
 
     # Convert DataFrame to a list of dictionaries (rows)
     rows = issues_df.to_dict(orient="records")
@@ -14,46 +22,56 @@ def get_issue_summary():
         {
             "display_severity": "No issues",
             "severity": "",
-            "total_count_percentage": 0.0,
+            "total_count": 0,
+            "total_count_percentage": 0.00,
             "internal_count": 0,
+            "internal_count_percentage": 0.00,
             "external_count": 0,
-            "total_count": 0,
+            "external_count_percentage": 0.00,
             "classes": "reporting-good-background",
         },
         {
             "display_severity": "Info",
             "severity": "info",
-            "total_count_percentage": 0.0,
+            "total_count": 0,
+            "total_count_percentage": 0.00,
             "internal_count": 0,
+            "internal_count_percentage": 0.00,
             "external_count": 0,
-            "total_count": 0,
+            "external_count_percentage": 0.00,
             "classes": "reporting-good-background",
         },
         {
             "display_severity": "Warning",
             "severity": "warning",
-            "total_count_percentage": 0.0,
+            "total_count": 0,
+            "total_count_percentage": 0.00,
             "internal_count": 0,
+            "internal_count_percentage": 0.00,
             "external_count": 0,
-            "total_count": 0,
+            "external_count_percentage": 0.00,
             "classes": "reporting-medium-background",
         },
         {
             "display_severity": "Error",
             "severity": "error",
-            "total_count_percentage": 0.0,
+            "total_count": 0,
+            "total_count_percentage": 0.00,
             "internal_count": 0,
+            "internal_count_percentage": 0.00,
             "external_count": 0,
-            "total_count": 0,
+            "external_count_percentage": 0.00,
             "classes": "reporting-bad-background",
         },
         {
             "display_severity": "Notice",
             "severity": "notice",
-            "total_count_percentage": 0.0,
+            "total_count": 0,
+            "total_count_percentage": 0.00,
             "internal_count": 0,
+            "internal_count_percentage": 0.00,
             "external_count": 0,
-            "total_count": 0,
+            "external_count_percentage": 0.00,
             "classes": "reporting-bad-background",
         },
     ]
@@ -93,53 +111,64 @@ def get_issue_summary():
     # Add issue_severity row
     stats_rows = []
     for issue_severity in issue_severity_counts:
+        if issue_severity["internal_count"] > 0:
+            issue_severity["internal_count_percentage"] = round(
+                (issue_severity["internal_count"] / total_issues) * 100, 2
+            )
+
+        if issue_severity["external_count"] > 0:
+            issue_severity["external_count_percentage"] = round(
+                (issue_severity["external_count"] / total_issues) * 100, 2
+            )
+
         if issue_severity["total_count"] > 0:
-            issue_severity[
-                "total_count_percentage"
-            ] = f"{int((issue_severity['total_count'] / total_issues) * 100)}%"
+            issue_severity["total_count_percentage"] = round(
+                (issue_severity["total_count"] / total_issues) * 100, 2
+            )
+
             stats_rows.append(
                 [
                     {
                         "text": issue_severity["display_severity"],
                         "classes": issue_severity["classes"] + " reporting-table-cell",
                     },
                     {
-                        "text": issue_severity["total_count"],
-                        "classes": "reporting-table-cell",
-                    },
-                    {
-                        "text": issue_severity["total_count_percentage"],
+                        "text": f"{issue_severity['internal_count']} ({issue_severity['internal_count_percentage']}%)",
                         "classes": "reporting-table-cell",
                     },
                     {
-                        "text": issue_severity["internal_count"],
+                        "text": f"{issue_severity['external_count']} ({issue_severity['external_count_percentage']}%)",
                         "classes": "reporting-table-cell",
                     },
                     {
-                        "text": issue_severity["external_count"],
+                        "text": f"{issue_severity['total_count']} ({issue_severity['total_count_percentage']}%)",
                         "classes": "reporting-table-cell",
                     },
                 ]
             )
 
-    # Add totals row
+    # Add totals row, the bottom
     stats_rows.append(
         [
             {"text": "Total", "classes": "reporting-table-cell"},
+            {
+                "text": f"{total_internal} ({round((total_internal/total_issues)*100, 2)}%)",
+                "classes": "reporting-table-cell",
+            },
+            {
+                "text": f"{total_external} ({round((total_external/total_issues)*100, 2)}%)",
+                "classes": "reporting-table-cell",
+            },
             {"text": total_issues, "classes": "reporting-table-cell"},
-            {"text": "", "classes": "reporting-table-cell"},
-            {"text": total_internal, "classes": "reporting-table-cell"},
-            {"text": total_external, "classes": "reporting-table-cell"},
         ]
     )
 
     # Define headers
     stats_headers = [
-        {"text": "Issue Severity"},
-        {"text": "Count"},
-        {"text": "% Count"},
-        {"text": "Internal"},
-        {"text": "External"},
+        {"text": "Severity"},
+        {"text": "Internal (%)"},
+        {"text": "External (%)"},
+        {"text": "Total (%)"},
     ]
 
     return {
@@ -153,16 +182,23 @@ def get_issue_summary():
     }
 
 
-def get_issue_summary_for_csv():
-    issue_summary_df = get_datasette_query_issue_summary(
-        "performance/endpoint_dataset_issue_type_summary"
-    )
+def get_full_issue_summary_for_csv():
+    pagination_incomplete = True
+    offset = 0
+    issue_summary_df_list = []
+    while pagination_incomplete:
+        issue_summary_df = get_issue_summary_for_csv(offset)
+        issue_summary_df_list.append(issue_summary_df)
+        pagination_incomplete = len(issue_summary_df) == 1000
+        offset += 1000
+    issue_summary_df = pd.concat(issue_summary_df_list)
+
     issue_summary_df = issue_summary_df[issue_summary_df["count_issues"].notna()]
 
     return issue_summary_df[
         [
             "organisation",
-            "name",
+            "organisation_name",
             "pipeline",
             "issue_type",
             "severity",
@@ -181,3 +217,17 @@ def get_issue_summary_for_csv():
             "resource_end_date",
         ]
     ]
+
+
+def get_issue_summary(offset):
+    sql = f"""
+    select count_issues, severity, responsibility from endpoint_dataset_issue_type_summary limit 1000 offset {offset}
+    """
+    return get_datasette_query("performance", sql)
+
+
+def get_issue_summary_for_csv(offset):
+    sql = f"""
+    select * from endpoint_dataset_issue_type_summary limit 1000 offset {offset}
+    """
+    return get_datasette_query("performance", sql)

application/templates/reporting/overview.html

@@ -36,18 +36,18 @@ <h2>Endpoint Collection Stats</h2>
       <p id="errors" class="reporting-clear-boths govuk-body">Endpoint errors: </p>
     </div>
     <div class="govuk-!-margin-top-6 govuk-!-width-full govuk-!-text-align-centre">
-      <a class="govuk-link" href="{{ url_for('reporting.download_csv')}}?type=issue-summary" id="download-button">
+      <a class="govuk-link" href="{{ url_for('reporting.download_csv')}}?type=endpoint_dataset_issue_type_summary" id="download-button">
       Download Issues Summary Table
       </a>
     </div>
 
   </div>
   <div class="govuk-grid-column-one-half govuk-!-padding-left-9">
-    <h1 class="govuk-heading-xl">Issues summary:</h1>
+    <h1 class="govuk-heading-xl">Issues:</h1>
       {{ govukTable({
         "head": issue_summary.stats_headers,
         "rows": issue_summary.stats_rows,
-        "classes": "reporting-table"
+        "classes": "reporting-table app-table-issue-summary"
       }) }}
       <p class="govuk-!-text-align-centre govuk-body govuk-!-width-full"><strong class="govuk-!-font-size-36">{{issue_summary.endpoints_no_issues.count}}</strong> / {{issue_summary.endpoints_no_issues.total_endpoints}} endpoints with no issues</p>
   </div>

src/scss/application.scss

@@ -44,7 +44,7 @@
 .app-summary-list__value--warning {
   padding-left: 10px;
   background: govuk-tint(govuk-colour("yellow"), 75);
-  border-left: 4px solid govuk-colour('yellow');
+  border-left: 4px solid govuk-colour("yellow");
 
   .app-summary-list__value__annotation {
     color: govuk-shade(govuk-colour("yellow"), 65);
@@ -64,7 +64,7 @@
 .app-summary-list__value--error {
   padding-left: 10px;
   background: govuk-tint(govuk-colour("red"), 80);
-  border-left: 4px solid govuk-colour('red');
+  border-left: 4px solid govuk-colour("red");
 
   .app-summary-list__value__annotation {
     color: govuk-shade(govuk-colour("red"), 30);
@@ -80,7 +80,7 @@
   padding: 10px;
   color: govuk-shade(govuk-colour("red"), 30);
   background: govuk-tint(govuk-colour("red"), 80);
-  border-left: 4px solid govuk-colour('red');
+  border-left: 4px solid govuk-colour("red");
 }
 
 .js-hidden {
@@ -105,3 +105,21 @@
 .app-main-wrapper {
   @include moj-width-container;
 }
+
+.app-table-issue-summary .govuk-table__header {
+  padding-top: 0;
+}
+
+.app-table-issue-summary .govuk-table__body {
+  white-space: normal;
+
+  @media (min-width: 71.0625em) {
+    white-space: nowrap;
+  }
+
+  .govuk-table__row:last-child {
+    .govuk-table__cell.reporting-table-cell {
+      font-weight: 700;
+    }
+  }
+}