Skip to content

Commit 98c39b9

Browse files
ceorourkeceorourke
authored
chore(ACI): Add metric for dual processing in metric alerts (#91979)
Follow up to #91840 The `organization_id` tag is not showing up in Datadog despite manually allowing it for these metrics (I suspect something higher up is blocking it) so this PR removes that tag and adds a metric behind the logging flag for metric alerts firing so that we can see if they are firing at the same rate as in workflow engine.
1 parent 89d2fbe commit 98c39b9

File tree

4 files changed

+37
-42
lines changed

4 files changed

+37
-42
lines changed

‎src/sentry/incidents/subscription_processor.py

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -368,16 +368,6 @@ def process_update(self, subscription_update: QuerySubscriptionUpdate) -> None:
368368
"organizations:anomaly-detection-alerts", organization
369369
) and features.has("organizations:anomaly-detection-rollout", organization)
370370

371-
alert_triggered_tags = {
372-
"detection_type": self.alert_rule.detection_type,
373-
"organization_id": None,
374-
}
375-
if features.has(
376-
"organizations:workflow-engine-metric-alert-dual-processing-logs",
377-
self.subscription.project.organization,
378-
):
379-
alert_triggered_tags["organization_id"] = self.alert_rule.organization_id
380-
381371
potential_anomalies = None
382372
if (
383373
has_anomaly_detection
@@ -440,7 +430,7 @@ def process_update(self, subscription_update: QuerySubscriptionUpdate) -> None:
440430
) and not self.check_trigger_matches_status(trigger, TriggerStatus.ACTIVE):
441431
metrics.incr(
442432
"incidents.alert_rules.threshold.alert",
443-
tags=alert_triggered_tags,
433+
tags={"detection_type": self.alert_rule.detection_type},
444434
)
445435
incident_trigger = self.trigger_alert_threshold(
446436
trigger, aggregation_value
@@ -457,7 +447,7 @@ def process_update(self, subscription_update: QuerySubscriptionUpdate) -> None:
457447
):
458448
metrics.incr(
459449
"incidents.alert_rules.threshold.resolve",
460-
tags=alert_triggered_tags,
450+
tags={"detection_type": self.alert_rule.detection_type},
461451
)
462452
incident_trigger = self.trigger_resolve_threshold(
463453
trigger, aggregation_value
@@ -479,8 +469,13 @@ def process_update(self, subscription_update: QuerySubscriptionUpdate) -> None:
479469
# And the trigger is not yet active
480470
metrics.incr(
481471
"incidents.alert_rules.threshold.alert",
482-
tags=alert_triggered_tags,
472+
tags={"detection_type": self.alert_rule.detection_type},
483473
)
474+
if features.has(
475+
"organizations:workflow-engine-metric-alert-dual-processing-logs",
476+
self.subscription.project.organization,
477+
):
478+
metrics.incr("dual_processing.alert_rules.fire")
484479
# triggering a threshold will create an incident and set the status to active
485480
incident_trigger = self.trigger_alert_threshold(trigger, aggregation_value)
486481
if incident_trigger is not None:
@@ -497,8 +492,13 @@ def process_update(self, subscription_update: QuerySubscriptionUpdate) -> None:
497492
):
498493
metrics.incr(
499494
"incidents.alert_rules.threshold.resolve",
500-
tags=alert_triggered_tags,
495+
tags={"detection_type": self.alert_rule.detection_type},
501496
)
497+
if features.has(
498+
"organizations:workflow-engine-metric-alert-dual-processing-logs",
499+
self.subscription.project.organization,
500+
):
501+
metrics.incr("dual_processing.alert_rules.fire")
502502
incident_trigger = self.trigger_resolve_threshold(
503503
trigger, aggregation_value
504504
)

‎src/sentry/workflow_engine/processors/workflow.py

Lines changed: 5 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import logging
22
from dataclasses import asdict, replace
33
from enum import StrEnum
4-
from typing import Any
54

65
import sentry_sdk
76
from django.db import router, transaction
@@ -218,14 +217,6 @@ def process_workflows(event_data: WorkflowEventData) -> set[Workflow]:
218217

219218
# TODO: remove fetching org, only used for feature flag checks
220219
organization = detector.project.organization
221-
workflow_metric_tags: dict[str, Any] = {
222-
"detector_type": detector.type,
223-
"organization_id": None,
224-
}
225-
if features.has(
226-
"organizations:workflow-engine-metric-alert-dual-processing-logs", organization
227-
):
228-
workflow_metric_tags["organization_id"] = organization.id
229220

230221
# Get the workflows, evaluate the when_condition_group, finally evaluate the actions for workflows that are triggered
231222
workflows = set(
@@ -240,7 +231,7 @@ def process_workflows(event_data: WorkflowEventData) -> set[Workflow]:
240231
metrics.incr(
241232
"workflow_engine.process_workflows",
242233
len(workflows),
243-
tags=workflow_metric_tags,
234+
tags={"detector_type": detector.type},
244235
)
245236

246237
logger.info(
@@ -263,7 +254,7 @@ def process_workflows(event_data: WorkflowEventData) -> set[Workflow]:
263254
metrics.incr(
264255
"workflow_engine.process_workflows.triggered_workflows",
265256
len(triggered_workflows),
266-
tags=workflow_metric_tags,
257+
tags={"detector_type": detector.type},
267258
)
268259

269260
logger.info(
@@ -284,7 +275,7 @@ def process_workflows(event_data: WorkflowEventData) -> set[Workflow]:
284275
metrics.incr(
285276
"workflow_engine.process_workflows.actions",
286277
amount=len(actions),
287-
tags=workflow_metric_tags,
278+
tags={"detector_type": detector.type},
288279
)
289280

290281
logger.info(
@@ -309,7 +300,7 @@ def process_workflows(event_data: WorkflowEventData) -> set[Workflow]:
309300
metrics.incr(
310301
"workflow_engine.process_workflows.triggered_actions",
311302
amount=len(actions),
312-
tags=workflow_metric_tags,
303+
tags={"detector_type": detector.type},
313304
)
314305
logger.info(
315306
"workflow_engine.process_workflows.triggered_actions (batch)",
@@ -324,7 +315,7 @@ def process_workflows(event_data: WorkflowEventData) -> set[Workflow]:
324315
# in order to check if workflow engine is firing 1:1 with the old system, we must only count once rather than each action
325316
metrics.incr(
326317
"workflow_engine.process_workflows.fired_actions",
327-
tags=workflow_metric_tags,
318+
tags={"detector_type": detector.type},
328319
)
329320

330321
return triggered_workflows

‎tests/sentry/incidents/test_subscription_processor.py

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1059,7 +1059,7 @@ def test_alert(self, create_metric_issue_mock, mock_metrics):
10591059
[
10601060
call(
10611061
"incidents.alert_rules.threshold.alert",
1062-
tags={"detection_type": "static", "organization_id": None},
1062+
tags={"detection_type": "static"},
10631063
),
10641064
call("incidents.alert_rules.trigger", tags={"type": "fire"}),
10651065
],
@@ -1075,7 +1075,10 @@ def test_alert_metrics(self, mock_metrics):
10751075
[
10761076
call(
10771077
"incidents.alert_rules.threshold.alert",
1078-
tags={"detection_type": "static", "organization_id": rule.organization_id},
1078+
tags={"detection_type": "static"},
1079+
),
1080+
call(
1081+
"dual_processing.alert_rules.fire",
10791082
),
10801083
call("incidents.alert_rules.trigger", tags={"type": "fire"}),
10811084
],
@@ -1249,12 +1252,12 @@ def test_resolve(self, create_metric_issue_mock, mock_metrics):
12491252
[
12501253
call(
12511254
"incidents.alert_rules.threshold.alert",
1252-
tags={"detection_type": "static", "organization_id": None},
1255+
tags={"detection_type": "static"},
12531256
),
12541257
call("incidents.alert_rules.trigger", tags={"type": "fire"}),
12551258
call(
12561259
"incidents.alert_rules.threshold.resolve",
1257-
tags={"detection_type": "static", "organization_id": None},
1260+
tags={"detection_type": "static"},
12581261
),
12591262
call("incidents.alert_rules.trigger", tags={"type": "resolve"}),
12601263
]
@@ -1271,12 +1274,18 @@ def test_resolve_metrics(self, mock_metrics):
12711274
[
12721275
call(
12731276
"incidents.alert_rules.threshold.alert",
1274-
tags={"detection_type": "static", "organization_id": rule.organization_id},
1277+
tags={"detection_type": "static"},
1278+
),
1279+
call(
1280+
"dual_processing.alert_rules.fire",
12751281
),
12761282
call("incidents.alert_rules.trigger", tags={"type": "fire"}),
12771283
call(
12781284
"incidents.alert_rules.threshold.resolve",
1279-
tags={"detection_type": "static", "organization_id": rule.organization_id},
1285+
tags={"detection_type": "static"},
1286+
),
1287+
call(
1288+
"dual_processing.alert_rules.fire",
12801289
),
12811290
call("incidents.alert_rules.trigger", tags={"type": "resolve"}),
12821291
]

‎tests/sentry/workflow_engine/processors/test_workflow.py

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,6 @@ def setUp(self):
6262
id=1, is_new=False, is_regression=True, is_new_group_environment=False
6363
),
6464
)
65-
self.workflow_metric_tags = {
66-
"detector_type": self.error_detector.type,
67-
"organization_id": None,
68-
}
6965

7066
def test_skips_disabled_workflows(self):
7167
workflow_triggers = self.create_data_condition_group()
@@ -280,7 +276,7 @@ def test_metrics_with_workflows(self, mock_incr):
280276
mock_incr.assert_any_call(
281277
"workflow_engine.process_workflows",
282278
1,
283-
tags=self.workflow_metric_tags,
279+
tags={"detector_type": self.error_detector.type},
284280
)
285281

286282
@patch("sentry.utils.metrics.incr")
@@ -290,7 +286,7 @@ def test_metrics_triggered_workflows(self, mock_incr):
290286
mock_incr.assert_any_call(
291287
"workflow_engine.process_workflows.triggered_workflows",
292288
1,
293-
tags=self.workflow_metric_tags,
289+
tags={"detector_type": self.error_detector.type},
294290
)
295291

296292
@with_feature("organizations:workflow-engine-process-workflows")
@@ -302,7 +298,7 @@ def test_metrics_triggered_actions(self, mock_incr):
302298
mock_incr.assert_any_call(
303299
"workflow_engine.process_workflows.triggered_actions",
304300
amount=0,
305-
tags=self.workflow_metric_tags,
301+
tags={"detector_type": self.error_detector.type},
306302
)
307303

308304
@with_feature("organizations:workflow-engine-process-workflows")
@@ -314,7 +310,6 @@ def test_metrics_issue_dual_processing_metrics(self, mock_incr):
314310
"workflow_engine.process_workflows.fired_actions",
315311
tags={
316312
"detector_type": self.error_detector.type,
317-
"organization_id": self.error_detector.project.organization_id,
318313
},
319314
)
320315

0 commit comments

Comments
 (0)