add ability to cache all assignment progress for a session

Moves the calculating of progress out to a method in a marking file,
uses that in a management command to store all progress for all types in
a pickle file and if that exists loads that on the front page.

This should significantly speed up the display of the front page for
admin users. For non admin users progress is loaded from the database as
before.

Author: struan

crowdsourcer/management/commands/cache_current_progress.py

@@ -0,0 +1,32 @@
+from django.core.management.base import BaseCommand
+
+from crowdsourcer.marking import (
+    get_assignment_progress,
+    save_cached_assignment_progress,
+)
+from crowdsourcer.models import Assigned, MarkingSession, ResponseType
+
+YELLOW = "\033[33m"
+NOBOLD = "\033[0m"
+
+
+class Command(BaseCommand):
+    help = "caches current progress for a stage and session to json"
+
+    def add_arguments(self, parser):
+        parser.add_argument("--session", action="store", help="name of the session")
+
+    def handle(self, session, *args, **kwargs):
+        ms = MarkingSession.objects.get(label=session)
+
+        for t in ResponseType.objects.all():
+            qs = Assigned.objects.filter(
+                marking_session=ms,
+                section__isnull=False,
+                active=True,
+                response_type=t,
+                user__is_active=True,
+            )
+
+            progress = get_assignment_progress(qs, session, t.type)
+            save_cached_assignment_progress(f"{session} {t.type}", progress)

crowdsourcer/marking.py

@@ -0,0 +1,125 @@
+import pickle
+
+from django.conf import settings
+from django.utils.text import slugify
+
+from crowdsourcer.models import (
+    Assigned,
+    MarkingSession,
+    PublicAuthority,
+    Question,
+    ResponseType,
+)
+
+
+def get_assignment_progress_cache_name(name):
+    name = slugify(name)
+    file = settings.BASE_DIR / "data" / f"assignment_progress_{name}.pkl"
+
+    return file
+
+
+def get_cached_assignment_progress(name):
+    file = get_assignment_progress_cache_name(name)
+
+    progress = None
+    if file.exists():
+        with open(file, "rb") as fp:
+            progress = pickle.load(fp)
+
+    return progress
+
+
+def save_cached_assignment_progress(name, progress):
+    file = get_assignment_progress_cache_name(name)
+    with open(file, "wb") as fp:
+        pickle.dump(progress, fp)
+
+
+def get_assignment_progress(assignments, marking_session, stage):
+    current_session = MarkingSession.objects.get(label=marking_session)
+
+    assignments = assignments.distinct(
+        "user_id", "section_id", "response_type_id"
+    ).select_related("section", "response_type")
+
+    types = Question.VOLUNTEER_TYPES
+    if stage == "Audit":
+        types = ["volunteer", "national_volunteer", "foi"]
+
+    first_mark = ResponseType.objects.get(type="First Mark")
+
+    progress = []
+    question_cache = {}
+    for assignment in assignments:
+        assignment_user = assignment.user
+        if hasattr(assignment_user, "marker"):
+            stage = assignment_user.marker.response_type
+        else:
+            stage = first_mark
+
+        if question_cache.get(assignment.section_id, None) is not None:
+            question_list = question_cache[assignment.section_id]
+        else:
+            questions = Question.objects.filter(
+                section=assignment.section, how_marked__in=types
+            )
+            question_list = list(questions.values_list("id", flat=True))
+            question_cache[assignment.section_id] = question_list
+
+        total = 0
+        complete = 0
+        started = 0
+
+        if assignment.section is not None:
+            args = [
+                question_list,
+                assignment.section.title,
+                assignment.user,
+                current_session,
+            ]
+            if assignment.authority_id is not None:
+                authorities = Assigned.objects.filter(
+                    active=True,
+                    user=assignment.user_id,
+                    section=assignment.section_id,
+                    response_type=stage,
+                ).values_list("authority_id", flat=True)
+                args.append(authorities)
+
+            # we pass the question list but we want to ignore it because there could be different types of council
+            # included in assignments which throws the count off
+            response_counts = PublicAuthority.response_counts(
+                *args,
+                question_types=types,
+                response_type=assignment.response_type,
+                ignore_question_list=True,
+            ).distinct()
+
+            for count in response_counts:
+                total += 1
+                if count.num_responses is not None and count.num_responses > 0:
+                    started += 1
+                if count.num_responses == count.num_questions:
+                    complete += 1
+
+        if assignment.response_type is None:
+            section_link = "home"
+        elif assignment.response_type.type == "First Mark":
+            section_link = "section_authorities"
+        elif assignment.response_type.type == "Right of Reply":
+            section_link = "authority_ror_authorities"
+        elif assignment.response_type.type == "Audit":
+            section_link = "audit_section_authorities"
+
+        progress.append(
+            {
+                "assignment": assignment,
+                "complete": complete,
+                "started": started,
+                "total": total,
+                "section_link": section_link,
+            }
+        )
+
+    return progress

crowdsourcer/views/marking.py

@@ -7,11 +7,11 @@
 from django.views.generic import FormView, ListView, TemplateView
 
 from crowdsourcer.forms import SessionPropertyForm
+from crowdsourcer.marking import get_assignment_progress, get_cached_assignment_progress
 from crowdsourcer.models import (
     Assigned,
     MarkingSession,
     PublicAuthority,
-    Question,
     ResponseType,
     SessionProperties,
     SessionPropertyValues,
@@ -157,89 +157,17 @@ def get_context_data(self, **kwargs):
 
             return context
 
-        assignments = (
-            context["assignments"]
-            .distinct("user_id", "section_id", "response_type_id")
-            .select_related("section", "response_type")
-        )
-
-        types = Question.VOLUNTEER_TYPES
-        if self.request.current_stage.type == "Audit":
-            types = ["volunteer", "national_volunteer", "foi"]
-
-        first_mark = ResponseType.objects.get(type="First Mark")
-
-        progress = []
-        question_cache = {}
-        for assignment in assignments:
-            assignment_user = assignment.user
-            if hasattr(assignment_user, "marker"):
-                stage = assignment_user.marker.response_type
-            else:
-                stage = first_mark
+        progress = None
+        if user.has_perm("crowdsourcer.can_view_all_responses"):
+            progress = get_cached_assignment_progress(
+                f"{self.request.current_session.label} {self.request.current_stage.type}"
+            )
 
-            if question_cache.get(assignment.section_id, None) is not None:
-                question_list = question_cache[assignment.section_id]
-            else:
-                questions = Question.objects.filter(
-                    section=assignment.section, how_marked__in=types
-                )
-                question_list = list(questions.values_list("id", flat=True))
-                question_cache[assignment.section_id] = question_list
-
-            total = 0
-            complete = 0
-            started = 0
-
-            if assignment.section is not None:
-                args = [
-                    question_list,
-                    assignment.section.title,
-                    assignment.user,
-                    self.request.current_session,
-                ]
-                if assignment.authority_id is not None:
-                    authorities = Assigned.objects.filter(
-                        active=True,
-                        user=assignment.user_id,
-                        section=assignment.section_id,
-                        response_type=stage,
-                    ).values_list("authority_id", flat=True)
-                    args.append(authorities)
-
-                # we pass the question list but we want to ignore it because there could be different types of council
-                # included in assignments which throws the count off
-                response_counts = PublicAuthority.response_counts(
-                    *args,
-                    question_types=types,
-                    response_type=assignment.response_type,
-                    ignore_question_list=True,
-                ).distinct()
-
-                for count in response_counts:
-                    total += 1
-                    if count.num_responses is not None and count.num_responses > 0:
-                        started += 1
-                    if count.num_responses == count.num_questions:
-                        complete += 1
-
-            if assignment.response_type is None:
-                section_link = "home"
-            elif assignment.response_type.type == "First Mark":
-                section_link = "section_authorities"
-            elif assignment.response_type.type == "Right of Reply":
-                section_link = "authority_ror_authorities"
-            elif assignment.response_type.type == "Audit":
-                section_link = "audit_section_authorities"
-
-            progress.append(
-                {
-                    "assignment": assignment,
-                    "complete": complete,
-                    "started": started,
-                    "total": total,
-                    "section_link": section_link,
-                }
+        if progress is None:
+            progress = get_assignment_progress(
+                context["assignments"],
+                self.request.current_session.label,
+                self.request.current_stage.type,
             )
 
         context["sessions"] = MarkingSession.objects.filter(active=True)