fix(hybrid-cloud): User deletion outboxes always fan out to all regions (#69363)

Author: GabeVillalobos

src/sentry/models/user.py

@@ -42,7 +42,7 @@
 from sentry.services.hybrid_cloud.organization import RpcRegionUser, organization_service
 from sentry.services.hybrid_cloud.user import RpcUser
 from sentry.types.integrations import EXTERNAL_PROVIDERS, ExternalProviders
-from sentry.types.region import find_regions_for_user
+from sentry.types.region import find_all_region_names, find_regions_for_user
 from sentry.utils.http import absolute_uri
 from sentry.utils.retries import TimedRetryPolicy
 
@@ -202,7 +202,7 @@ def delete(self):
             avatar = self.avatar.first()
             if avatar:
                 avatar.delete()
-            for outbox in self.outboxes_for_update():
+            for outbox in self.outboxes_for_update(is_user_delete=True):
                 outbox.save()
             return super().delete()
 
@@ -305,13 +305,23 @@ def send_confirm_emails(self, is_new_user=False):
         for email in email_list:
             self.send_confirm_email_singular(email, is_new_user)
 
-    def outboxes_for_update(self) -> list[ControlOutboxBase]:
-        return User.outboxes_for_user_update(self.id)
+    def outboxes_for_update(self, is_user_delete: bool = False) -> list[ControlOutboxBase]:
+        return User.outboxes_for_user_update(self.id, is_user_delete=is_user_delete)
 
     @staticmethod
-    def outboxes_for_user_update(identifier: int) -> list[ControlOutboxBase]:
+    def outboxes_for_user_update(
+        identifier: int, is_user_delete: bool = False
+    ) -> list[ControlOutboxBase]:
+        # User deletions must fan out to all regions to ensure cascade behavior
+        # of anything with a HybridCloudForeignKey, even if the user is no longer
+        # a member of any organizations in that region.
+        if is_user_delete:
+            user_regions = set(find_all_region_names())
+        else:
+            user_regions = find_regions_for_user(identifier)
+
         return OutboxCategory.USER_UPDATE.as_control_outboxes(
-            region_names=find_regions_for_user(identifier),
+            region_names=user_regions,
             object_identifier=identifier,
             shard_identifier=identifier,
         )

tests/sentry/models/test_user.py

@@ -9,8 +9,8 @@
 from sentry.testutils.cases import TestCase
 from sentry.testutils.hybrid_cloud import HybridCloudTestMixin
 from sentry.testutils.outbox import outbox_runner
-from sentry.testutils.silo import assume_test_silo_mode, control_silo_test
-from sentry.types.region import Region, RegionCategory
+from sentry.testutils.silo import assume_test_silo_mode, assume_test_silo_mode_of, control_silo_test
+from sentry.types.region import Region, RegionCategory, find_regions_for_user
 
 _TEST_REGIONS = (
     Region("na", 1, "http://eu.testserver", RegionCategory.MULTI_TENANT),
@@ -34,21 +34,21 @@ def setUp(self):
         )
 
     @assume_test_silo_mode(SiloMode.REGION)
-    def user_tombstone_exists(self) -> bool:
+    def user_tombstone_exists(self, user_id: int) -> bool:
         return RegionTombstone.objects.filter(
-            table_name="auth_user", object_identifier=self.user_id
+            table_name="auth_user", object_identifier=user_id
         ).exists()
 
     @assume_test_silo_mode(SiloMode.REGION)
     def get_user_saved_search_count(self) -> int:
         return SavedSearch.objects.filter(owner_id=self.user_id).count()
 
     def test_simple(self):
-        assert not self.user_tombstone_exists()
+        assert not self.user_tombstone_exists(user_id=self.user_id)
         with outbox_runner():
             self.user.delete()
         assert not User.objects.filter(id=self.user_id).exists()
-        assert self.user_tombstone_exists()
+        assert self.user_tombstone_exists(user_id=self.user_id)
 
         # cascade is asynchronous, ensure there is still related search,
         assert self.get_user_saved_search_count() == 1
@@ -87,6 +87,35 @@ def test_cascades_to_multiple_regions(self):
             schedule_hybrid_cloud_foreign_key_jobs()
         assert self.get_user_saved_search_count() == 0
 
+    def test_deletions_create_tombstones_in_regions_for_user_with_no_orgs(self):
+        # Create a user with no org memberships
+        user_to_delete = self.create_user("foo@example.com")
+        user_id = user_to_delete.id
+        with outbox_runner():
+            user_to_delete.delete()
+
+        assert self.user_tombstone_exists(user_id=user_id)
+
+    def test_cascades_to_regions_even_if_user_ownership_revoked(self):
+        eu_org = self.create_organization(region=_TEST_REGIONS[1])
+        self.create_member(user=self.user, organization=eu_org)
+        self.create_saved_search(name="eu-search", owner=self.user, organization=eu_org)
+        assert self.get_user_saved_search_count() == 2
+
+        with outbox_runner(), assume_test_silo_mode_of(OrganizationMember):
+            for member in OrganizationMember.objects.filter(user_id=self.user.id):
+                member.delete()
+
+        assert find_regions_for_user(self.user.id) == set()
+
+        with outbox_runner():
+            self.user.delete()
+
+        assert self.get_user_saved_search_count() == 2
+        with assume_test_silo_mode(SiloMode.REGION), self.tasks():
+            schedule_hybrid_cloud_foreign_key_jobs()
+        assert self.get_user_saved_search_count() == 0
+
 
 @control_silo_test
 class UserDetailsTest(TestCase):