feat(features) Add metrics to feature flag decisions (#68554)

I'd like to get baseline data for the current duration of feature flag
checks and success ratios. In the future I'd like to have alerting on
feature flags that are always on for extended periods of time.

I've set the sample rate to 1% to keep overhead low as feature flags are
a high throughput code path.

Refs HC-848

Author: markstory

src/sentry/features/manager.py

@@ -12,6 +12,8 @@
 import sentry_sdk
 from django.conf import settings
 
+from sentry.utils import metrics
+
 from .base import Feature, FeatureHandlerStrategy
 from .exceptions import FeatureNotRegistered
 
@@ -210,26 +212,49 @@ def has(self, name: str, *args: Any, skip_entity: bool | None = False, **kwargs:
         >>> FeatureManager.has('organizations:feature', organization, actor=request.user)
 
         """
+        sample_rate = 0.01
         try:
-            actor = kwargs.pop("actor", None)
-            feature = self.get(name, *args, **kwargs)
+            with metrics.timer("features.has", tags={"name": name}, sample_rate=sample_rate):
+                actor = kwargs.pop("actor", None)
+                feature = self.get(name, *args, **kwargs)
 
-            # Check registered feature handlers
-            rv = self._get_handler(feature, actor)
-            if rv is not None:
-                return rv
+                # Check registered feature handlers
+                rv = self._get_handler(feature, actor)
+                if rv is not None:
+                    metrics.incr(
+                        "feature.has.result",
+                        tags={"name": name, "result": rv},
+                        sample_rate=sample_rate,
+                    )
+                    return rv
 
-            if self._entity_handler and not skip_entity:
-                rv = self._entity_handler.has(feature, actor)
+                if self._entity_handler and not skip_entity:
+                    rv = self._entity_handler.has(feature, actor)
+                    if rv is not None:
+                        metrics.incr(
+                            "feature.has.result",
+                            tags={"name": name, "result": rv},
+                            sample_rate=sample_rate,
+                        )
+                        return rv
+
+                rv = settings.SENTRY_FEATURES.get(feature.name, False)
                 if rv is not None:
+                    metrics.incr(
+                        "feature.has.result",
+                        tags={"name": name, "result": rv},
+                        sample_rate=sample_rate,
+                    )
                     return rv
 
-            rv = settings.SENTRY_FEATURES.get(feature.name, False)
-            if rv is not None:
-                return rv
+                # Features are by default disabled if no plugin or default enables them
+                metrics.incr(
+                    "feature.has.result",
+                    tags={"name": name, "result": False},
+                    sample_rate=sample_rate,
+                )
 
-            # Features are by default disabled if no plugin or default enables them
-            return False
+                return False
         except Exception:
             logging.exception("Failed to run feature check")
             return False