Merge pull request #34 from digital-land/overview-of-datasets-update

minor Overview of datasets summary update

Author: kena-SL

application/data_access/overview/digital_land_queries.py

@@ -172,6 +172,7 @@ def get_overall_source_counts(groupby=None):
         "source",
         "INNER JOIN source_pipeline on source.source = source_pipeline.source",
         "INNER JOIN organisation on organisation.organisation = replace(source.organisation,'-eng','')",
+        "WHERE source.end_date is null or source.end_date ==''",
         (
             f"GROUP BY {groupby_options.get(groupby)}"
             if groupby and groupby_options.get(groupby)
@@ -193,7 +194,7 @@ def get_grouped_source_counts(organisation=None, **kwargs):
 
 
 def fetch_total_resource_count():
-    sql = "select count(distinct resource) from resource"
+    sql = "select count(distinct resource) from resource where end_date is null or end_date ==''"
     rows = get_datasette_query("digital-land", f"""{sql}""")
 
     return rows.iloc[0][0] if len(rows) > 0 else 0

application/data_access/overview/entity_queries.py

@@ -1,3 +1,6 @@
+import json
+import urllib.request
+
 from application.data_access.datasette_utils import get_datasette_query
 from application.data_access.overview.api_queries import get_organisation_entity_number
 from application.utils import split_organisation_id
@@ -32,19 +35,11 @@ def get_grouped_entity_count(dataset=None, organisation_entity=None):
     return {}
 
 
-def get_total_entity_count():
-    sql = "select count(*) from lookup"
-    row = get_datasette_query("digital-land", sql)
-
-    return row.iloc[0][0] if len(row) > 0 else 0
-
-
 def get_entity_count(pipeline=None):
-    if pipeline is not None:
-        sql = "select count(*) from lookup WHERE prefix = :pipeline"
-        row = get_datasette_query("digital-land", sql, {"pipeline": pipeline})
-        return row.iloc[0][0] if len(row) > 0 else 0
-    return get_total_entity_count()
+    url = f"https://www.planning.data.gov.uk/entity.json{f'?prefix={pipeline}' if pipeline else ''}"
+    with urllib.request.urlopen(url) as response:
+        data = json.load(response)
+        return data["count"]
 
 
 def get_organisation_entity_count(organisation, dataset=None):

application/templates/overview/performance.html

@@ -42,7 +42,7 @@ <h2>{{ entity_count|commanum }}</h2>
                   <div class="govuk-grid-column-one-quarter">
                     <div class="app-data-item">
                       <p class="govuk-body app-data-item__label">Datasets</p>
-                      <h2>{{ datasets_with_data_count }}<h2">/{{ datasets|length }}</h2></h2>
+                      <h2>{{ datasets|length }}</h2>
                     </div>
                   </div>
                   <div class="govuk-grid-column-one-quarter">