refac(discord): Update metrics and logging (#68819)

Update how we send metrics and handle logging during Discord responses.
Using more standard metric events to alleviate logic handling in
monitoring platforms. Success events end with `"success"`, and failures
events with `"failure"`.

We currently need to ignore specific user errors as a result of us not
turning off base alert actions. Some alert actions are broken, because
the user has deleted the channel/guild/team, or entered an improper id,
but because these continue to execute, we continue to log and put them
as part of our metrics. When the main problem is fixed, we will not see
so many user errors come through.

Resolves:
https://github.com/getsentry/team-core-product-foundations/issues/218#issue-2200762819

Author: ykamo001

src/sentry/integrations/discord/client.py

@@ -2,6 +2,7 @@
 
 import logging
 from collections.abc import Mapping
+from typing import Any
 from urllib.parse import urlencode
 
 from rest_framework import status
@@ -43,6 +44,9 @@
 class DiscordClient(ApiClient):
     integration_name: str = "discord"
     base_url: str = DISCORD_BASE_URL
+    _METRICS_FAILURE_KEY: str = "sentry.integrations.discord.failure"
+    _METRICS_SUCCESS_KEY: str = "sentry.integrations.discord.success"
+    _METRICS_USER_ERROR_KEY: str = "sentry.integrations.discord.failure.user_error"
 
     def __init__(self):
         super().__init__()
@@ -118,13 +122,22 @@ def track_response_data(
         extra: Mapping[str, str] | None = None,
     ) -> None:
         """
-        For all Discord api responses this:
-        - Sends response metrics to Datadog
-        - Sends response info to logs
+        Handle response from Discord by logging and capturing metrics
         """
-        discord_error_code = "no_error_code"
-        code_message = ""
-        include_in_slo = True
+        log_params = {
+            "code": code,
+            "error": str(error),
+            "extra": extra,
+        }
+
+        if self.integration_type:
+            log_params[str(self.integration_type)] = self.name
+
+        try:
+            logging_context = getattr(self, "logging_context", None)
+            log_params["logging_context"] = logging_context
+        except Exception:
+            pass
 
         is_ok = code in {
             status.HTTP_200_OK,
@@ -133,44 +146,63 @@ def track_response_data(
             status.HTTP_204_NO_CONTENT,
         }
 
-        if error:
-            try:
-                discord_error_response: dict = json.loads(resp.content.decode("utf-8")) or {}  # type: ignore[union-attr]
-                discord_error_code = str(discord_error_response.get("code", ""))
-                if discord_error_code in DISCORD_ERROR_CODES:
-                    code_message = DISCORD_ERROR_CODES[discord_error_code]
-                # These are excluded since they are not actionable from our side
-                if discord_error_code in DISCORD_USER_ERRORS:
-                    include_in_slo = False
-            except Exception:
-                pass
+        if not is_ok or error:
+            self._handle_failure(log_params=log_params, resp=resp)
+        else:
+            self._handle_success(log_params=log_params)
+
+    def _handle_failure(
+        self,
+        log_params: dict[str, Any],
+        resp: Response | None = None,
+    ) -> None:
+        """
+        Do extra logic to handle an error from Discord
+        """
 
+        discord_error_response: dict | None = None
+        if resp is not None:
+            # Try to get the additional error code that Discord sent us to help determine what specific error happened
+            try:
+                discord_error_response = json.loads(resp.content.decode("utf-8"))
+                log_params["discord_error_response"] = discord_error_response
+            except Exception as err:
+                self.logger.info(
+                    "error trying to handle discord error message", exc_info=err, extra=log_params
+                )
+
+        discord_error_code = None
+        if discord_error_response is not None:
+            # Discord sends us a special code for errors in the response data
+            # https://discord.com/developers/docs/topics/opcodes-and-status-codes#json
+            discord_error_code = str(discord_error_response.get("code", ""))
+            log_params["discord_error_code"] = discord_error_code
+
+            # Get the specific meaning for those codes
+            if discord_error_code_message := DISCORD_ERROR_CODES.get(discord_error_code, None):
+                log_params["discord_error_code_message"] = discord_error_code_message
+
+        # Check if the error is due to a user configuration error, which we do not have control over to fix
+        # An example of this would be if the user deleted the discord guild and never updated the alert action
+        is_user_error = discord_error_code in DISCORD_USER_ERRORS
+        log_params["is_user_error"] = is_user_error
+
+        metrics_key = self._METRICS_USER_ERROR_KEY if is_user_error else self._METRICS_FAILURE_KEY
         metrics.incr(
-            f"{self.metrics_prefix}.http_response",
+            metrics_key,
             sample_rate=1.0,
-            tags={
-                str(self.integration_type): self.name,
-                "status": code,
-                "is_ok": is_ok,
-                "include_in_slo": include_in_slo,
-                "discord_code": discord_error_code,
-            },
         )
+        self.logger.info("handled discord error", extra=log_params)
 
-        log_params = {
-            **(extra or {}),
-            "status_string": str(code),
-            "error": str(error)[:256] if error else None,
-            "include_in_slo": include_in_slo,
-            "discord_code": discord_error_code,
-            "code_message": code_message if error else None,
-        }
-
-        if self.integration_type:
-            log_params[self.integration_type] = self.name
-
-        log_params.update(getattr(self, "logging_context", None) or {})
-        self.logger.info("%s.http_response", self.integration_type, extra=log_params)
+    def _handle_success(
+        self,
+        log_params: dict[str, Any],
+    ) -> None:
+        metrics.incr(
+            self._METRICS_SUCCESS_KEY,
+            sample_rate=1.0,
+        )
+        self.logger.info("handled discord success", extra=log_params)
 
     def send_message(
         self, channel_id: str, message: DiscordMessageBuilder, notification_uuid: str | None = None