chore(issues): Log details on very old time-to-post-process (#68590)

We are seeing a handful of time-to-post-process metrics recorded which
have very large durations, in some cases >1 month. Logging some
additional information on these groups should help us investigate more.

The test here is a bit overkill but for now ensures that we're not
accidentally logging on every single group.

Author: mrduncan

src/sentry/tasks/post_process.py

@@ -747,13 +747,23 @@ def get_event_raise_exception() -> Event:
             metric_tags["occurrence_type"] = group_event.group.issue_type.slug
 
         if not is_reprocessed and event.data.get("received"):
+            duration = time() - event.data["received"]
             metrics.timing(
                 "events.time-to-post-process",
-                time() - event.data["received"],
+                duration,
                 instance=event.data["platform"],
                 tags=metric_tags,
             )
 
+            # We see occasional metrics being recorded with very old data,
+            # temporarily log some information about these groups to help
+            # investigate.
+            if duration and duration > 604_800:  # 7 days (7*24*60*60)
+                logger.warning(
+                    "tasks.post_process.old_time_to_post_process",
+                    extra={"group_id": group_id, "project_id": project_id, "duration": duration},
+                )
+
 
 def run_post_process_job(job: PostProcessJob):
     group_event = job["event"]

tests/sentry/tasks/test_post_process.py

@@ -186,6 +186,24 @@ def test_processing_cache_cleared_with_commits(self):
         )
         assert event_processing_store.get(cache_key) is None
 
+    @patch("sentry.utils.metrics.timing")
+    @patch("sentry.tasks.post_process.logger")
+    def test_time_to_process_metric(self, logger_mock, metric_timing_mock):
+        event = self.create_event(data={}, project_id=self.project.id)
+        self.call_post_process_group(
+            is_new=True,
+            is_regression=False,
+            is_new_group_environment=True,
+            event=event,
+        )
+        metric_timing_mock.assert_any_call(
+            "events.time-to-post-process",
+            mock.ANY,
+            instance=mock.ANY,
+            tags={"occurrence_type": mock.ANY},
+        )
+        logger_mock.warning.assert_not_called()
+
 
 class DeriveCodeMappingsProcessGroupTestMixin(BasePostProgressGroupMixin):
     def _create_event(