Improve logic for automatically updating part pricing (#8090) (#8122)

* Improve logic for automatically updating part pricing

* Simplify logic

* Update unit tests to ensure pricing flows upwards

* Unit test update

* Add unit tests for handling of "multi level" BOM pricing

* ADjust unit tests

* Improve efficiency of operation

* Adjust testing for pricing

- Only allow pricing updates in testing if TESTING_PRICING flag is set

* Tweak when pricing updates are performed

* More tweaks

Author: SchrodingersGat

src/backend/InvenTree/InvenTree/settings.py

@@ -1284,6 +1284,9 @@
 # Flag to allow table events during testing
 TESTING_TABLE_EVENTS = False
 
+# Flag to allow pricing recalculations during testing
+TESTING_PRICING = False
+
 # User interface customization values
 CUSTOM_LOGO = get_custom_file(
     'INVENTREE_CUSTOM_LOGO', 'customize.logo', 'custom logo', lookup_media=True

src/backend/InvenTree/InvenTree/unit_test.py

@@ -264,7 +264,7 @@ class InvenTreeAPITestCase(ExchangeRateMixin, UserMixin, APITestCase):
     MAX_QUERY_TIME = 7.5
 
     @contextmanager
-    def assertNumQueriesLessThan(self, value, using='default', verbose=None, url=None):
+    def assertNumQueriesLessThan(self, value, using='default', verbose=False, url=None):
         """Context manager to check that the number of queries is less than a certain value.
 
         Example:
@@ -282,10 +282,8 @@ def assertNumQueriesLessThan(self, value, using='default', verbose=None, url=Non
                 f'Query count exceeded at {url}: Expected < {value} queries, got {n}'
             )  # pragma: no cover
 
-        if verbose or n >= value:
-            msg = '\r\n%s' % json.dumps(
-                context.captured_queries, indent=4
-            )  # pragma: no cover
+        if verbose and n >= value:
+            msg = f'\r\n{json.dumps(context.captured_queries, indent=4)}'  # pragma: no cover
         else:
             msg = None
 

src/backend/InvenTree/build/test_api.py

@@ -966,6 +966,12 @@ def setUpTestData(cls):
         outputs = cls.build.build_outputs.all()
         cls.build.complete_build_output(outputs[0], cls.user)
 
+    def setUp(self):
+        """Basic operation as part of test suite setup"""
+        super().setUp()
+
+        self.generate_exchange_rates()
+
     def test_setup(self):
         """Validate expected state after set-up."""
         self.assertEqual(self.build.incomplete_outputs.count(), 0)
@@ -994,7 +1000,7 @@ def test_overallocated_requires_acceptance(self):
                 'accept_overallocated': 'accept',
             },
             expected_code=201,
-            max_query_count=550,  # TODO: Come back and refactor this
+            max_query_count=1000,  # TODO: Come back and refactor this
         )
 
         self.build.refresh_from_db()
@@ -1015,9 +1021,11 @@ def test_overallocated_can_trim(self):
                 'accept_overallocated': 'trim',
             },
             expected_code=201,
-            max_query_count=555,  # TODO: Come back and refactor this
+            max_query_count=1000,  # TODO: Come back and refactor this
         )
 
+        # Note: Large number of queries is due to pricing recalculation for each stock item
+
         self.build.refresh_from_db()
 
         # Build should have been marked as complete

src/backend/InvenTree/company/models.py

@@ -1052,7 +1052,10 @@ def get_api_url():
 )
 def after_save_supplier_price(sender, instance, created, **kwargs):
     """Callback function when a SupplierPriceBreak is created or updated."""
-    if InvenTree.ready.canAppAccessDatabase() and not InvenTree.ready.isImportingData():
+    if (
+        InvenTree.ready.canAppAccessDatabase(allow_test=settings.TESTING_PRICING)
+        and not InvenTree.ready.isImportingData()
+    ):
         if instance.part and instance.part.part:
             instance.part.part.schedule_pricing_update(create=True)
 
@@ -1064,6 +1067,9 @@ def after_save_supplier_price(sender, instance, created, **kwargs):
 )
 def after_delete_supplier_price(sender, instance, **kwargs):
     """Callback function when a SupplierPriceBreak is deleted."""
-    if InvenTree.ready.canAppAccessDatabase() and not InvenTree.ready.isImportingData():
+    if (
+        InvenTree.ready.canAppAccessDatabase(allow_test=settings.TESTING_PRICING)
+        and not InvenTree.ready.isImportingData()
+    ):
         if instance.part and instance.part.part:
             instance.part.part.schedule_pricing_update(create=False)

src/backend/InvenTree/part/models.py

@@ -1931,7 +1931,7 @@ def pricing(self):
 
         return pricing
 
-    def schedule_pricing_update(self, create: bool = False, test: bool = False):
+    def schedule_pricing_update(self, create: bool = False):
         """Helper function to schedule a pricing update.
 
         Importantly, catches any errors which may occur during deletion of related objects,
@@ -1941,7 +1941,6 @@ def schedule_pricing_update(self, create: bool = False, test: bool = False):
 
         Arguments:
             create: Whether or not a new PartPricing object should be created if it does not already exist
-            test: Whether or not the pricing update is allowed during unit tests
         """
         try:
             self.refresh_from_db()
@@ -1952,7 +1951,7 @@ def schedule_pricing_update(self, create: bool = False, test: bool = False):
             pricing = self.pricing
 
             if create or pricing.pk:
-                pricing.schedule_for_update(test=test)
+                pricing.schedule_for_update()
         except IntegrityError:
             # If this part instance has been deleted,
             # some post-delete or post-save signals may still be fired
@@ -2532,7 +2531,8 @@ class PartPricing(common.models.MetaMixin):
     - Detailed pricing information is very context specific in any case
     """
 
-    price_modified = False
+    # When calculating assembly pricing, we limit the depth of the calculation
+    MAX_PRICING_DEPTH = 50
 
     @property
     def is_valid(self):
@@ -2561,14 +2561,10 @@ def convert(self, money):
 
         return result
 
-    def schedule_for_update(self, counter: int = 0, test: bool = False):
+    def schedule_for_update(self, counter: int = 0):
         """Schedule this pricing to be updated."""
         import InvenTree.ready
 
-        # If we are running within CI, only schedule the update if the test flag is set
-        if settings.TESTING and not test:
-            return
-
         # If importing data, skip pricing update
         if InvenTree.ready.isImportingData():
             return
@@ -2612,7 +2608,7 @@ def schedule_for_update(self, counter: int = 0, test: bool = False):
             logger.debug('Pricing for %s already scheduled for update - skipping', p)
             return
 
-        if counter > 25:
+        if counter > self.MAX_PRICING_DEPTH:
             # Prevent infinite recursion / stack depth issues
             logger.debug(
                 counter, f'Skipping pricing update for {p} - maximum depth exceeded'
@@ -2631,16 +2627,36 @@ def schedule_for_update(self, counter: int = 0, test: bool = False):
 
         import part.tasks as part_tasks
 
+        # Pricing calculations are performed in the background,
+        # unless the TESTING_PRICING flag is set
+        background = not settings.TESTING or not settings.TESTING_PRICING
+
         # Offload task to update the pricing
-        # Force async, to prevent running in the foreground
+        # Force async, to prevent running in the foreground (unless in testing mode)
         InvenTree.tasks.offload_task(
-            part_tasks.update_part_pricing, self, counter=counter, force_async=True
+            part_tasks.update_part_pricing,
+            self,
+            counter=counter,
+            force_async=background,
         )
 
-    def update_pricing(self, counter: int = 0, cascade: bool = True):
-        """Recalculate all cost data for the referenced Part instance."""
-        # If importing data, skip pricing update
+    def update_pricing(
+        self,
+        counter: int = 0,
+        cascade: bool = True,
+        previous_min=None,
+        previous_max=None,
+    ):
+        """Recalculate all cost data for the referenced Part instance.
+
+        Arguments:
+            counter: Recursion counter (used to prevent infinite recursion)
+            cascade: If True, update pricing for all assemblies and templates which use this part
+            previous_min: Previous minimum price (used to prevent further updates if unchanged)
+            previous_max: Previous maximum price (used to prevent further updates if unchanged)
 
+        """
+        # If importing data, skip pricing update
         if InvenTree.ready.isImportingData():
             return
 
@@ -2671,18 +2687,25 @@ def update_pricing(self, counter: int = 0, cascade: bool = True):
             # Background worker processes may try to concurrently update
             pass
 
+        pricing_changed = False
+
+        # Without previous pricing data, we assume that the pricing has changed
+        if previous_min != self.overall_min or previous_max != self.overall_max:
+            pricing_changed = True
+
         # Update parent assemblies and templates
-        if cascade and self.price_modified:
+        if pricing_changed and cascade:
             self.update_assemblies(counter)
             self.update_templates(counter)
 
     def update_assemblies(self, counter: int = 0):
         """Schedule updates for any assemblies which use this part."""
         # If the linked Part is used in any assemblies, schedule a pricing update for those assemblies
+
         used_in_parts = self.part.get_used_in()
 
         for p in used_in_parts:
-            p.pricing.schedule_for_update(counter + 1)
+            p.pricing.schedule_for_update(counter=counter + 1)
 
     def update_templates(self, counter: int = 0):
         """Schedule updates for any template parts above this part."""
@@ -2698,13 +2721,13 @@ def save(self, *args, **kwargs):
 
         try:
             self.update_overall_cost()
-        except IntegrityError:
+        except Exception:
             # If something has happened to the Part model, might throw an error
             pass
 
         try:
             super().save(*args, **kwargs)
-        except IntegrityError:
+        except Exception:
             # This error may be thrown if there is already duplicate pricing data
             pass
 
@@ -2772,9 +2795,6 @@ def update_bom_cost(self, save=True):
 
                 any_max_elements = True
 
-        old_bom_cost_min = self.bom_cost_min
-        old_bom_cost_max = self.bom_cost_max
-
         if any_min_elements:
             self.bom_cost_min = cumulative_min
         else:
@@ -2785,12 +2805,6 @@ def update_bom_cost(self, save=True):
         else:
             self.bom_cost_max = None
 
-        if (
-            old_bom_cost_min != self.bom_cost_min
-            or old_bom_cost_max != self.bom_cost_max
-        ):
-            self.price_modified = True
-
         if save:
             self.save()
 
@@ -2854,12 +2868,6 @@ def update_purchase_cost(self, save=True):
                 if purchase_max is None or cost > purchase_max:
                     purchase_max = cost
 
-        if (
-            self.purchase_cost_min != purchase_min
-            or self.purchase_cost_max != purchase_max
-        ):
-            self.price_modified = True
-
         self.purchase_cost_min = purchase_min
         self.purchase_cost_max = purchase_max
 
@@ -2886,12 +2894,6 @@ def update_internal_cost(self, save=True):
                 if max_int_cost is None or cost > max_int_cost:
                     max_int_cost = cost
 
-        if (
-            self.internal_cost_min != min_int_cost
-            or self.internal_cost_max != max_int_cost
-        ):
-            self.price_modified = True
-
         self.internal_cost_min = min_int_cost
         self.internal_cost_max = max_int_cost
 
@@ -2927,12 +2929,6 @@ def update_supplier_cost(self, save=True):
                     if max_sup_cost is None or cost > max_sup_cost:
                         max_sup_cost = cost
 
-        if (
-            self.supplier_price_min != min_sup_cost
-            or self.supplier_price_max != max_sup_cost
-        ):
-            self.price_modified = True
-
         self.supplier_price_min = min_sup_cost
         self.supplier_price_max = max_sup_cost
 
@@ -2968,9 +2964,6 @@ def update_variant_cost(self, save=True):
                     if variant_max is None or v_max > variant_max:
                         variant_max = v_max
 
-        if self.variant_cost_min != variant_min or self.variant_cost_max != variant_max:
-            self.price_modified = True
-
         self.variant_cost_min = variant_min
         self.variant_cost_max = variant_max
 
@@ -3091,12 +3084,6 @@ def update_sale_cost(self, save=True):
             if max_sell_history is None or cost > max_sell_history:
                 max_sell_history = cost
 
-        if (
-            self.sale_history_min != min_sell_history
-            or self.sale_history_max != max_sell_history
-        ):
-            self.price_modified = True
-
         self.sale_history_min = min_sell_history
         self.sale_history_max = max_sell_history
 
@@ -4509,7 +4496,10 @@ def update_bom_build_lines(sender, instance, created, **kwargs):
 def update_pricing_after_edit(sender, instance, created, **kwargs):
     """Callback function when a part price break is created or updated."""
     # Update part pricing *unless* we are importing data
-    if InvenTree.ready.canAppAccessDatabase() and not InvenTree.ready.isImportingData():
+    if (
+        InvenTree.ready.canAppAccessDatabase(allow_test=settings.TESTING_PRICING)
+        and not InvenTree.ready.isImportingData()
+    ):
         if instance.part:
             instance.part.schedule_pricing_update(create=True)
 
@@ -4526,7 +4516,10 @@ def update_pricing_after_edit(sender, instance, created, **kwargs):
 def update_pricing_after_delete(sender, instance, **kwargs):
     """Callback function when a part price break is deleted."""
     # Update part pricing *unless* we are importing data
-    if InvenTree.ready.canAppAccessDatabase() and not InvenTree.ready.isImportingData():
+    if (
+        InvenTree.ready.canAppAccessDatabase(allow_test=settings.TESTING_PRICING)
+        and not InvenTree.ready.isImportingData()
+    ):
         if instance.part:
             instance.part.schedule_pricing_update(create=False)
 

src/backend/InvenTree/part/tasks.py

@@ -73,7 +73,11 @@ def update_part_pricing(pricing: part.models.PartPricing, counter: int = 0):
     """
     logger.info('Updating part pricing for %s', pricing.part)
 
-    pricing.update_pricing(counter=counter)
+    pricing.update_pricing(
+        counter=counter,
+        previous_min=pricing.overall_min,
+        previous_max=pricing.overall_max,
+    )
 
 
 @scheduled_task(ScheduledTask.DAILY)