fix(structure): correctly update structure in-place for recalculations

When calculating a field, we must take better care to update
the dependents, invalidate all the cached values, and ensure the
intermediate values are correctly used.

Author: winged

caluma/caluma_form/structure.py

@@ -18,6 +18,7 @@
 
 import collections
 import copy
+import itertools
 import typing
 import weakref
 from abc import ABC
@@ -175,9 +176,13 @@ def _store_question(self, question):
                 continue
             # Evaluator is only used for extracting answer transforms & friends,
             # so is field-independent here
-            for dependency_slug in self._evaluator.extract_referenced_questions(
-                jexl_expr
-            ):
+            referenced_questions = set(
+                itertools.chain(
+                    self._evaluator.extract_referenced_questions(jexl_expr),
+                    self._evaluator.extract_referenced_mapby_questions(jexl_expr),
+                )
+            )
+            for dependency_slug in referenced_questions:
                 self._jexl_dependencies[dependency_slug][question.pk].append(
                     expr_property
                 )
@@ -553,6 +558,12 @@ def refresh(self, answer=None):
         """
         if answer:
             self.answer = answer
+        elif self.answer:
+            self.answer.refresh_from_db()
+        elif not isinstance(self, FieldSet):
+            self.answer = Answer.objects.filter(
+                question=self.question, document=self.parent._document
+            ).first()
 
         clear_memoise(self)
         for dep in self._fastloader.dependents_of_question(self.slug()):
@@ -754,7 +765,6 @@ def is_empty(self):
     def get_context(self):
         return self._context
 
-    @object_local_memoise
     def get_value(self):
         if self.is_empty():
             # is_empty() will return True if we're hidden, so

caluma/caluma_form/tests/test_question.py

@@ -1167,7 +1167,7 @@ def test_recalc_missing_dependency(
     sub_question.type = models.Question.TYPE_INTEGER
     sub_question.save()
 
-    spy = mocker.spy(domain_logic.SaveAnswerLogic, "update_calc_dependents")
+    spy = mocker.spy(domain_logic.SaveAnswerLogic, "recalculate_dependents")
 
     # Calculated question in another form
     form_question_factory(

caluma/caluma_form/tests/test_structure.py

@@ -5,6 +5,7 @@
 import pytest
 
 from caluma.caluma_form import structure
+from caluma.caluma_form.api import save_answer
 from caluma.caluma_form.models import Answer, Document, FormQuestion, Question
 
 
@@ -422,3 +423,60 @@ def test_fastloader_no_duplicate_options(
     assert len(the_options) == len(
         struc._fastloader.options_for_question(choice_q.slug)
     )
+
+
+def test_multistage_calculation_updates(
+    simple_form_structure,
+    form_factory,
+    django_assert_num_queries,
+    form_question_factory,
+):
+    """Verify recalculation behaviour across multiple calc stages.
+
+    A chain of calculations across tables is done to ensure we get the right
+    values at each step and don't end up in a cache trap
+    """
+    form_question_factory(
+        form_id="root",
+        question__type=Question.TYPE_CALCULATED_FLOAT,
+        question__slug="outer-calc",
+        question__calc_expression="'sub_table'|answer([])|mapby('row_calc')|sum",
+    )
+
+    struc0 = structure.FieldSet(simple_form_structure)
+
+    assert struc0._fastloader._questions["leaf2"].calc_dependents == ["row_calc"]
+    assert struc0._fastloader._questions["row_field_2"].calc_dependents == ["row_calc"]
+    assert struc0._fastloader._questions["row_calc"].calc_dependents == ["outer-calc"]
+    assert struc0._fastloader._questions["outer-calc"].calc_dependents == []
+
+    for field in struc0.find_all_fields_by_slug("row_field_2"):
+        answer = save_answer(
+            field.question, field.answer.document, value=field.answer.value + 2
+        )
+        field.refresh(answer)
+
+    struc1 = structure.FieldSet(simple_form_structure)
+
+    # The struc0 should be internally-refreshed, so must match another
+    # structure constructed freshly from DB
+    assert struc0.list_structure() == struc1.list_structure()
+
+    assert struc1.list_structure() == [
+        " FieldSet(root)",
+        "    Field(leaf1, Some Value)",
+        "    Field(leaf2, 33)",
+        "    FieldSet(measure-evening)",
+        "       Field(sub_leaf1, None)",
+        "       Field(sub_leaf2, None)",
+        "       RowSet(too-wonder-option)",
+        "          FieldSet(too-wonder-option)",
+        "             Field(row_field_1, 2025-01-13)",
+        "             Field(row_field_2, 101.5)",
+        "             Field(row_calc, 134.5)",
+        "          FieldSet(too-wonder-option)",
+        "             Field(row_field_1, 2025-01-10)",
+        "             Field(row_field_2, 25.0)",
+        "             Field(row_calc, 58.0)",
+        "    Field(outer-calc, 192.5)",
+    ]