feat(crons): Record broken monitor recovery analytic (#69260)

Records the event from #69259
when a monitor environment with a broken detection is resolved

Author: David Wang

src/sentry/monitors/logic/mark_ok.py

@@ -1,7 +1,11 @@
 import logging
-from datetime import datetime
+from datetime import datetime, timedelta
 
+from django.utils import timezone
+
+from sentry import analytics
 from sentry.monitors.models import CheckInStatus, MonitorCheckIn, MonitorEnvironment, MonitorStatus
+from sentry.monitors.tasks.detect_broken_monitor_envs import NUM_DAYS_BROKEN_PERIOD
 
 logger = logging.getLogger(__name__)
 
@@ -59,6 +63,18 @@ def mark_ok(checkin: MonitorCheckIn, ts: datetime):
                         "grouphash": incident.grouphash,
                     },
                 )
+                # if incident was longer than the broken env time, check if there was a broken detection that is also now resolved
+                if incident.starting_timestamp <= timezone.now() - timedelta(
+                    days=NUM_DAYS_BROKEN_PERIOD
+                ):
+                    if incident.monitorenvbrokendetection_set.exists():
+                        analytics.record(
+                            "cron_monitor_broken_status.recovery",
+                            organization_id=monitor_env.monitor.organization_id,
+                            project_id=monitor_env.monitor.project_id,
+                            monitor_id=monitor_env.monitor.id,
+                            monitor_env_id=monitor_env.id,
+                        )
 
     MonitorEnvironment.objects.filter(id=monitor_env.id).exclude(last_checkin__gt=ts).update(
         **params

tests/sentry/monitors/logic/test_mark_ok.py

@@ -1,4 +1,5 @@
 from datetime import timedelta
+from unittest import mock
 from unittest.mock import patch
 
 from django.utils import timezone
@@ -10,6 +11,7 @@
     CheckInStatus,
     Monitor,
     MonitorCheckIn,
+    MonitorEnvBrokenDetection,
     MonitorEnvironment,
     MonitorIncident,
     MonitorStatus,
@@ -240,3 +242,72 @@ def test_mark_ok_recovery_threshold(self, mock_produce_occurrence_to_kafka):
                 "new_substatus": None,
             },
         ) == dict(status_change)
+
+    @mock.patch("sentry.analytics.record")
+    def test_mark_ok_broken_recovery(self, mock_record):
+        now = timezone.now().replace(second=0, microsecond=0)
+
+        monitor = Monitor.objects.create(
+            name="test monitor",
+            organization_id=self.organization.id,
+            project_id=self.project.id,
+            type=MonitorType.CRON_JOB,
+            config={
+                "schedule": "* * * * *",
+                "schedule_type": ScheduleType.CRONTAB,
+                "max_runtime": None,
+                "checkin_margin": None,
+                "recovery_threshold": None,
+            },
+        )
+
+        # Start with monitor in an ERROR state and broken detection
+        monitor_environment = MonitorEnvironment.objects.create(
+            monitor=monitor,
+            environment_id=self.environment.id,
+            status=MonitorStatus.ERROR,
+            last_checkin=now - timedelta(minutes=1),
+            next_checkin=now,
+        )
+        checkin = MonitorCheckIn.objects.create(
+            monitor=monitor,
+            monitor_environment=monitor_environment,
+            project_id=self.project.id,
+            status=CheckInStatus.ERROR,
+            date_added=timezone.now() - timedelta(days=14),
+        )
+        incident = MonitorIncident.objects.create(
+            monitor=monitor,
+            monitor_environment=monitor_environment,
+            starting_checkin=checkin,
+            starting_timestamp=checkin.date_added,
+        )
+        MonitorEnvBrokenDetection.objects.create(
+            monitor_incident=incident,
+        )
+
+        # OK checkin comes in
+        success_checkin = MonitorCheckIn.objects.create(
+            monitor=monitor,
+            monitor_environment=monitor_environment,
+            project_id=self.project.id,
+            status=CheckInStatus.OK,
+            date_added=now,
+        )
+        mark_ok(success_checkin, ts=now)
+
+        # Monitor has recovered to OK with updated upcoming timestamps
+        monitor_environment.refresh_from_db()
+        assert monitor_environment.status == MonitorStatus.OK
+        assert monitor_environment.next_checkin == now + timedelta(minutes=1)
+        assert monitor_environment.next_checkin_latest == now + timedelta(minutes=2)
+        assert monitor_environment.last_checkin == now
+
+        # We recorded an analytics event
+        mock_record.assert_called_with(
+            "cron_monitor_broken_status.recovery",
+            organization_id=self.organization.id,
+            project_id=self.project.id,
+            monitor_id=monitor.id,
+            monitor_env_id=monitor_environment.id,
+        )