fix(seer grouping): Store events with unusable Seer matches in Seer (#89318)

When we send an event to Seer during ingest, if Seer doesn't find a match, it stores the event's data in its database, so that future events can match with it. Conversely, if Seer does find a match, it doesn't store the new event's data, under the presumption that any future events which would match it would also match the event it matched.*
  
Now that we're sending events with hybrid fingerprints to Seer, though, that logic breaks down, since an event being a stacktrace distance match doesn't guarantee that it's an overall match, because either or both could have a hybrid fingerprint, and the non-default parts of those potential hybrid fingerprints might or might not match. In other words, we can no longer assume that the event already in Seer and the new event are equivalent in terms of matching to future events. 

If neither the current event nor the existing event in Seer has a hybrid fingerprint, or if they both do _and_ the non-default parts of their fingerprints match, the assumption holds. But if neither of those is the case, future events could match one but not the other, meaning both need to be represented in the Seer database.

This PR thus adds a second request to Seer to store the new event in the case where none of the results returned by Seer are an overall match. In order to force the event to be stored, the request sets the desired number of matches to 0, so that Seer sees the event as one with no matches. It also turns off re-ranking, in order to prevent Seer from pulling an initial list of matches to whittle down, since we know we're not going to use any of them anyway.


*In terms of stacktrace distance, this is not perfectly theoretically true - something just barely close enough to the new event to count might be just slightly too far from the existing event - but in practice those corner cases are rare enough that we don't worry about them.

Author: lobsterkatie

src/sentry/grouping/ingest/seer.py

@@ -323,6 +323,30 @@ def get_seer_similar_issues(
                 seer_match_status = "match_found"
                 break
 
+    # If Seer sent back matches, that means it didn't store the incoming event's data in its
+    # database. But if we then weren't able to use any of the matches Seer sent back, we do actually
+    # want a Seer record to be created, so that future events with this fingerprint have something
+    # with which to match.
+    if seer_match_status == "no_matches_usable":
+        request_data = {
+            **request_data,
+            "referrer": "ingest_follow_up",
+            # By asking Seer to find zero matches, we can trick it into thinking there aren't
+            # any, thereby forcing it to create the record
+            "k": 0,
+            # Turn off re-ranking to speed up the process of finding nothing
+            "use_reranking": False,
+        }
+
+        # TODO: Temporary log to prove things are working as they should. This should come in a pair
+        # with the `get_similarity_data_from_seer.ingest_follow_up` log in `similar_issues.py`,
+        # which should show that no matches are returned.
+        logger.info("get_seer_similar_issues.follow_up_seer_request", extra={"hash": event_hash})
+
+        # We only want this for the side effect, and we know it'll return no matches, so we don't
+        # bother to capture the return value.
+        get_similarity_data_from_seer(request_data)
+
     is_hybrid_fingerprint_case = (
         event_has_hybrid_fingerprint
         # This means we had to reject at least one match because it was a hybrid even though the

src/sentry/seer/similarity/similar_issues.py

@@ -134,6 +134,17 @@ def get_similarity_data_from_seer(
         )
         return []
 
+    # TODO: Temporary log to prove things are working as they should. This should come in a pair
+    # with the `get_seer_similar_issues.follow_up_seer_request` log in `seer.py`.
+    if referrer == "ingest_follow_up":
+        logger.info(
+            "get_similarity_data_from_seer.ingest_follow_up",
+            extra={
+                "hash": request_hash,
+                "response_data": response_data,  # Should always be an empty list
+            },
+        )
+
     if not response_data:
         metrics.incr(
             "seer.similar_issues_request",

tests/sentry/grouping/seer_similarity/test_get_seer_similar_issues.py

@@ -1,5 +1,5 @@
 from typing import Any
-from unittest.mock import MagicMock, patch
+from unittest.mock import MagicMock, call, patch
 
 from sentry import options
 from sentry.eventstore.models import Event
@@ -112,6 +112,72 @@ def test_sends_expected_data_to_seer(self, mock_get_similarity_data: MagicMock)
             {"hybrid_fingerprint": False},
         )
 
+    @patch("sentry.grouping.ingest.seer.metrics.incr")
+    def test_sends_second_seer_request_when_seer_matches_are_unusable(
+        self, mock_incr: MagicMock
+    ) -> None:
+
+        existing_event = save_new_event(
+            {"message": "Dogs are great!", "fingerprint": ["{{ default }}", "maisey"]},
+            self.project,
+        )
+        assert existing_event.group_id
+
+        new_event, new_variants, new_grouphash, new_stacktrace_string = create_new_event(
+            self.project
+        )
+
+        seer_result_data = [
+            SeerSimilarIssueData(
+                parent_hash=existing_event.get_primary_hash(),
+                parent_group_id=existing_event.group_id,
+                stacktrace_distance=0.01,
+                should_group=True,
+            )
+        ]
+
+        with patch(
+            "sentry.grouping.ingest.seer.get_similarity_data_from_seer",
+            return_value=seer_result_data,
+        ) as mock_get_similarity_data:
+            get_seer_similar_issues(new_event, new_grouphash, new_variants)
+
+            assert_metrics_call(
+                mock_incr, "get_seer_similar_issues", "no_matches_usable", {"is_hybrid": True}
+            )
+
+            base_request_params = {
+                "event_id": new_event.event_id,
+                "hash": new_event.get_primary_hash(),
+                "project_id": self.project.id,
+                "stacktrace": new_stacktrace_string,
+                "exception_type": "FailedToFetchError",
+            }
+
+            assert mock_get_similarity_data.call_count == 2
+            assert mock_get_similarity_data.mock_calls == [
+                # Initial call to Seer
+                call(
+                    {
+                        **base_request_params,
+                        "k": options.get("seer.similarity.ingest.num_matches_to_request"),
+                        "referrer": "ingest",
+                        "use_reranking": True,
+                    },
+                    {"hybrid_fingerprint": False},
+                ),
+                # Second call to store the event's data since the match that came back from Seer
+                # wasn't usable
+                call(
+                    {
+                        **base_request_params,
+                        "k": 0,
+                        "referrer": "ingest_follow_up",
+                        "use_reranking": False,
+                    }
+                ),
+            ]
+
 
 class ParentGroupFoundTest(TestCase):
     @patch("sentry.grouping.ingest.seer.metrics.distribution")