Merge pull request #51 from digital-land/internal_error_graph_weekly

Internal Error Day to Week

Author: averheecke-tpx

application/blueprints/report/views.py

@@ -50,7 +50,7 @@
     get_contributions_and_errors_by_day,
     get_endpoint_errors_and_successes_by_week,
     get_endpoints_added_by_week,
-    get_internal_issues_by_day,
+    get_internal_issues_by_week,
     get_issue_counts,
 )
 from application.utils import (
@@ -80,7 +80,7 @@ def overview():
         endpoint_errors_percentages_timeseries,
     ) = get_endpoint_errors_and_successes_by_week(contributions_and_errors_by_day_df)
 
-    internal_errors_timeseries = get_internal_issues_by_day()
+    internal_errors_timeseries = get_internal_issues_by_week()
 
     summary_metrics = {
         "contributions": summary_contributions,

application/data_access/summary_queries.py

@@ -55,11 +55,16 @@ def get_issue_counts():
         return None
 
 
-def get_internal_issues_by_day():
-    sql = """SELECT COUNT(*) as count, [entry-date] as date
-             FROM operational_issue
-             GROUP BY [entry-date]
-             ORDER BY date ASC"""
+def get_internal_issues_by_week():
+    """
+    returns a list of weeks and unknown entity counts,
+    to include "invalid organisation" when integrated into table
+    """
+    sql = """SELECT COUNT(*) as count, [entry-date] as date, "issue-type"
+            FROM operational_issue
+            WHERE "issue-type" = "unknown entity"
+            GROUP BY [entry-date]
+            ORDER BY date ASC"""
 
     internal_issues_df = get_datasette_query("digital-land", sql)
 
@@ -70,20 +75,36 @@ def get_internal_issues_by_day():
     start_date = datetime(2024, 9, 27).date()
     end_date = datetime.now(timezone.utc).date()
 
-    all_days = pd.DataFrame({"date": pd.date_range(start=start_date, end=end_date)})
-
-    # Ensure datetime for the 'date' column for merge
-    all_days["date"] = pd.to_datetime(all_days["date"])
     internal_issues_df["date"] = pd.to_datetime(internal_issues_df["date"])
+    # set start of the week to Monday
+    internal_issues_df["week_start"] = internal_issues_df["date"] - pd.to_timedelta(
+        internal_issues_df["date"].dt.dayofweek, unit="D"
+    )
+
+    weekly_issues = (
+        internal_issues_df.groupby("week_start")["count"].sum().reset_index()
+    )
+
+    all_weeks = pd.DataFrame(
+        {
+            "week_start": pd.date_range(
+                start=start_date - pd.Timedelta(days=start_date.weekday()),
+                end=end_date,
+                freq="W-MON",
+            )
+        }
+    )
+
+    all_weeks_issues = pd.merge(all_weeks, weekly_issues, on="week_start", how="left")
 
-    all_days_issues = pd.merge(all_days, internal_issues_df, on="date", how="left")
+    all_weeks_issues["count"].fillna(0, inplace=True)
 
-    all_days_issues["count"].fillna(0, inplace=True)
+    all_weeks_issues.rename(columns={"week_start": "date"}, inplace=True)
 
-    all_days_issues["date"] = all_days_issues["date"].dt.strftime("%Y-%m-%d")
+    all_weeks_issues["date"] = all_weeks_issues["date"].dt.strftime("%Y-%m-%d")
 
-    internal_issues = all_days_issues.to_dict(orient="records")
-    return internal_issues
+    weekly_issues_list = all_weeks_issues.to_dict(orient="records")
+    return weekly_issues_list
 
 
 def get_contributions_and_erroring_endpoints(contributions_and_errors_df):

application/templates/reporting/overview.html

@@ -69,7 +69,7 @@ <h2>Endpoint errors and successes per week:</h2>
   <canvas id="endpoint_error_graph"></canvas>
 </div>
 
-<h2>Interal errors per day:</h2>
+<h2>Interal errors per week:</h2>
 <div>
   <canvas id="internal_error_graph"></canvas>
 </div>
@@ -138,10 +138,10 @@ <h2>Interal errors per day:</h2>
   const interal_error_timeseries = new TimeseriesChart(
     {
       ...baseOptions,
-      xAxisTitle: "Day",
+      xAxisTitle: "Week",
       yAxisTitle: "Internal errors",
       datasets: [{
-                label: "# of internal errors per day",
+                label: "# of internal errors per week",
                 data: {{graphs.internal_errors_timeseries | safe}},
                 borderWidth: 1,
                 pointRadius: 0