From 227dd78336c8203b6c9a43e71976c7d9ff74cbc4 Mon Sep 17 00:00:00 2001 From: Colleen O'Rourke Date: Wed, 28 May 2025 12:58:18 -0700 Subject: [PATCH 01/11] ref(ACI): Update evaluate_value return --- .../workflow_engine/models/data_condition.py | 5 +- .../test_anomaly_detection_handler.py | 173 ++++++++++++++++++ 2 files changed, 177 insertions(+), 1 deletion(-) create mode 100644 tests/sentry/incidents/handlers/condition/test_anomaly_detection_handler.py diff --git a/src/sentry/workflow_engine/models/data_condition.py b/src/sentry/workflow_engine/models/data_condition.py index 258986a85283bc..7a58464ff212f4 100644 --- a/src/sentry/workflow_engine/models/data_condition.py +++ b/src/sentry/workflow_engine/models/data_condition.py @@ -209,7 +209,10 @@ def evaluate_value(self, value: T) -> DataConditionResult: ) return None - return self.get_condition_result() if result else None + if isinstance(result, bool): + return self.get_condition_result() if result else None + + return result def is_slow_condition(condition: DataCondition) -> bool: diff --git a/tests/sentry/incidents/handlers/condition/test_anomaly_detection_handler.py b/tests/sentry/incidents/handlers/condition/test_anomaly_detection_handler.py new file mode 100644 index 00000000000000..cd21dbd9e57bef --- /dev/null +++ b/tests/sentry/incidents/handlers/condition/test_anomaly_detection_handler.py @@ -0,0 +1,173 @@ +from datetime import UTC, datetime +from unittest import mock + +import orjson +import pytest +from urllib3.response import HTTPResponse + +from sentry.incidents.handlers.condition.anomaly_detection_handler import DetectorError +from sentry.incidents.utils.types import MetricDetectorUpdate +from sentry.seer.anomaly_detection.types import ( + AnomalyDetectionSeasonality, + AnomalyDetectionSensitivity, + AnomalyDetectionThresholdType, + AnomalyType, + DataSourceType, + DetectAnomaliesResponse, +) +from sentry.snuba.subscriptions import create_snuba_subscription +from sentry.workflow_engine.models import Condition, DataPacket +from sentry.workflow_engine.types import DetectorPriorityLevel +from tests.sentry.workflow_engine.handlers.condition.test_base import ConditionTestCase + + +class TestAnomalyDetectionHandler(ConditionTestCase): + condition = Condition.ANOMALY_DETECTION + + def setUp(self): + super().setUp() + self.snuba_query = self.create_snuba_query() + self.subscription = create_snuba_subscription(self.project, "test", self.snuba_query) + + (self.workflow, self.detector, self.detector_workflow, self.workflow_triggers) = ( + self.create_detector_and_workflow() + ) + + subscription_update: MetricDetectorUpdate = { + "subscription_id": str(self.subscription.id), + "values": {"value": 1}, + "timestamp": datetime.now(UTC), + "entity": "test-entity", + } + + self.data_source = self.create_data_source( + source_id=str(subscription_update["subscription_id"]), + organization=self.organization, + ) + self.data_source.detectors.add(self.detector) + + self.data_packet = DataPacket[MetricDetectorUpdate]( + source_id=str(subscription_update["subscription_id"]), + packet=subscription_update, + ) + + self.dc = self.create_data_condition( + type=self.condition, + comparison={ + "sensitivity": AnomalyDetectionSensitivity.MEDIUM, + "seasonality": AnomalyDetectionSeasonality.AUTO, + "threshold_type": AnomalyDetectionThresholdType.ABOVE_AND_BELOW, + }, + condition_result=DetectorPriorityLevel.HIGH, + condition_group=self.workflow_triggers, + ) + + @mock.patch( + "sentry.seer.anomaly_detection.get_anomaly_data.SEER_ANOMALY_DETECTION_CONNECTION_POOL.urlopen" + ) + def test_passes(self, mock_seer_request): + seer_return_value: DetectAnomaliesResponse = { + "success": True, + "timeseries": [ + { + "anomaly": { + "anomaly_score": 0.9, + "anomaly_type": AnomalyType.HIGH_CONFIDENCE, + }, + "timestamp": 1, + "value": 10, + } + ], + } + mock_seer_request.return_value = HTTPResponse(orjson.dumps(seer_return_value), status=200) + self.assert_passes(self.dc, self.data_packet) + + @mock.patch( + "sentry.seer.anomaly_detection.get_anomaly_data.SEER_ANOMALY_DETECTION_CONNECTION_POOL.urlopen" + ) + def test_passes_medium(self, mock_seer_request): + seer_return_value: DetectAnomaliesResponse = { + "success": True, + "timeseries": [ + { + "anomaly": { + "anomaly_score": 0.2, + "anomaly_type": AnomalyType.LOW_CONFIDENCE, + }, + "timestamp": 1, + "value": 10, + } + ], + } + mock_seer_request.return_value = HTTPResponse(orjson.dumps(seer_return_value), status=200) + print("tset result: ", self.dc.evaluate_value(self.data_packet)) + assert self.dc.evaluate_value(self.data_packet) == DetectorPriorityLevel.OK.value + + @mock.patch( + "sentry.seer.anomaly_detection.get_anomaly_data.SEER_ANOMALY_DETECTION_CONNECTION_POOL.urlopen" + ) + @mock.patch("sentry.seer.anomaly_detection.get_anomaly_data.logger") + def test_seer_call_timeout_error(self, mock_logger, mock_seer_request): + from urllib3.exceptions import TimeoutError + + mock_seer_request.side_effect = TimeoutError + timeout_extra = { + "subscription_id": self.subscription.id, + "organization_id": self.organization.id, + "project_id": self.project.id, + "source_id": self.subscription.id, + "source_type": DataSourceType.SNUBA_QUERY_SUBSCRIPTION, + "dataset": self.subscription.snuba_query.dataset, + } + with pytest.raises(DetectorError): + self.dc.evaluate_value(self.data_packet) + mock_logger.warning.assert_called_with( + "Timeout error when hitting anomaly detection endpoint", extra=timeout_extra + ) + + @mock.patch( + "sentry.seer.anomaly_detection.get_anomaly_data.SEER_ANOMALY_DETECTION_CONNECTION_POOL.urlopen" + ) + @mock.patch("sentry.seer.anomaly_detection.get_anomaly_data.logger") + def test_seer_call_empty_list(self, mock_logger, mock_seer_request): + seer_return_value: DetectAnomaliesResponse = {"success": True, "timeseries": []} + mock_seer_request.return_value = HTTPResponse(orjson.dumps(seer_return_value), status=200) + with pytest.raises(DetectorError): + self.dc.evaluate_value(self.data_packet) + assert mock_logger.warning.call_args[0] == ( + "Seer anomaly detection response returned no potential anomalies", + ) + + @mock.patch( + "sentry.seer.anomaly_detection.get_anomaly_data.SEER_ANOMALY_DETECTION_CONNECTION_POOL.urlopen" + ) + @mock.patch("sentry.seer.anomaly_detection.get_anomaly_data.logger") + def test_seer_call_bad_status(self, mock_logger, mock_seer_request): + mock_seer_request.return_value = HTTPResponse(status=403) + extra = { + "subscription_id": self.subscription.id, + "organization_id": self.organization.id, + "project_id": self.project.id, + "source_id": self.subscription.id, + "source_type": DataSourceType.SNUBA_QUERY_SUBSCRIPTION, + "dataset": self.subscription.snuba_query.dataset, + "response_data": None, + } + with pytest.raises(DetectorError): + self.dc.evaluate_value(self.data_packet) + mock_logger.error.assert_called_with( + "Error when hitting Seer detect anomalies endpoint", extra=extra + ) + + @mock.patch( + "sentry.seer.anomaly_detection.get_anomaly_data.SEER_ANOMALY_DETECTION_CONNECTION_POOL.urlopen" + ) + @mock.patch("sentry.seer.anomaly_detection.get_anomaly_data.logger") + def test_seer_call_failed_parse(self, mock_logger, mock_seer_request): + # XXX: coercing a response into something that will fail to parse + mock_seer_request.return_value = HTTPResponse(None, status=200) # type: ignore[arg-type] + with pytest.raises(DetectorError): + self.dc.evaluate_value(self.data_packet) + mock_logger.exception.assert_called_with( + "Failed to parse Seer anomaly detection response", extra=mock.ANY + ) From 6b58f3f85e294e305e3880e208015ae82ad5d78a Mon Sep 17 00:00:00 2001 From: Colleen O'Rourke Date: Wed, 28 May 2025 13:28:00 -0700 Subject: [PATCH 02/11] add test and bare minimum for anomaly detection --- src/sentry/seer/anomaly_detection/types.py | 29 ++++++++++++++- .../handlers/condition/__init__.py | 2 + .../condition/anomaly_detection_handler.py | 37 ++++++++++++++++--- .../models/test_data_condition.py | 25 ++++++++++++- 4 files changed, 84 insertions(+), 9 deletions(-) diff --git a/src/sentry/seer/anomaly_detection/types.py b/src/sentry/seer/anomaly_detection/types.py index af45ffef4b65a0..48bba49da8dd59 100644 --- a/src/sentry/seer/anomaly_detection/types.py +++ b/src/sentry/seer/anomaly_detection/types.py @@ -1,4 +1,4 @@ -from enum import Enum +from enum import Enum, IntEnum, StrEnum from typing import NotRequired, TypedDict @@ -74,3 +74,30 @@ class AnomalyType(Enum): LOW_CONFIDENCE = "anomaly_lower_confidence" NONE = "none" NO_DATA = "no_data" + + +class AnomalyDetectionSensitivity(StrEnum): + LOW = "low" + MEDIUM = "medium" + HIGH = "high" + + +class AnomalyDetectionSeasonality(StrEnum): + """All combinations of multi select fields for anomaly detection alerts + We do not anticipate adding more + """ + + AUTO = "auto" + HOURLY = "hourly" + DAILY = "daily" + WEEKLY = "weekly" + HOURLY_DAILY = "hourly_daily" + HOURLY_WEEKLY = "hourly_weekly" + HOURLY_DAILY_WEEKLY = "hourly_daily_weekly" + DAILY_WEEKLY = "daily_weekly" + + +class AnomalyDetectionThresholdType(IntEnum): + ABOVE = 0 + BELOW = 1 + ABOVE_AND_BELOW = 2 diff --git a/src/sentry/workflow_engine/handlers/condition/__init__.py b/src/sentry/workflow_engine/handlers/condition/__init__.py index 5f79c87f30d0d9..2b4d0d53b66487 100644 --- a/src/sentry/workflow_engine/handlers/condition/__init__.py +++ b/src/sentry/workflow_engine/handlers/condition/__init__.py @@ -1,5 +1,6 @@ __all__ = [ "AgeComparisonConditionHandler", + "AnomalyDetectionHandler", "AssignedToConditionHandler", "EventAttributeConditionHandler", "EventCreatedByDetectorConditionHandler", @@ -26,6 +27,7 @@ ] from .age_comparison_handler import AgeComparisonConditionHandler +from .anomaly_detection_handler import AnomalyDetectionHandler from .assigned_to_handler import AssignedToConditionHandler from .event_attribute_handler import EventAttributeConditionHandler from .event_created_by_detector_handler import EventCreatedByDetectorConditionHandler diff --git a/src/sentry/workflow_engine/handlers/condition/anomaly_detection_handler.py b/src/sentry/workflow_engine/handlers/condition/anomaly_detection_handler.py index 16dd1ddb6e4430..3de0f14c4384e0 100644 --- a/src/sentry/workflow_engine/handlers/condition/anomaly_detection_handler.py +++ b/src/sentry/workflow_engine/handlers/condition/anomaly_detection_handler.py @@ -1,16 +1,41 @@ -from typing import Any - +from sentry.seer.anomaly_detection.types import ( + AnomalyDetectionSeasonality, + AnomalyDetectionSensitivity, + AnomalyDetectionThresholdType, +) from sentry.workflow_engine.models.data_condition import Condition from sentry.workflow_engine.registry import condition_handler_registry -from sentry.workflow_engine.types import DataConditionHandler, WorkflowEventData +from sentry.workflow_engine.types import ( + DataConditionHandler, + DetectorPriorityLevel, + WorkflowEventData, +) @condition_handler_registry.register(Condition.ANOMALY_DETECTION) class AnomalyDetectionHandler(DataConditionHandler[WorkflowEventData]): group = DataConditionHandler.Group.DETECTOR_TRIGGER - comparison_json_schema = {"type": "boolean"} + comparison_json_schema = { + "type": "object", + "properties": { + "sensitivity": { + "type": "string", + "enum": [*AnomalyDetectionSensitivity], + }, + "seasonality": { + "type": "string", + "enum": [*AnomalyDetectionSeasonality], + }, + "threshold_type": { + "type": "integer", + "enum": [*AnomalyDetectionThresholdType], + }, + }, + "required": ["sensitivity", "seasonality", "threshold_type"], + "additionalProperties": False, + } @staticmethod - def evaluate_value(event_data: WorkflowEventData, comparison: Any) -> bool: + def evaluate_value(event_data: WorkflowEventData, comparison: int) -> DetectorPriorityLevel: # this is a placeholder - return False + return DetectorPriorityLevel.HIGH if event_data > 1 else DetectorPriorityLevel.OK diff --git a/tests/sentry/workflow_engine/models/test_data_condition.py b/tests/sentry/workflow_engine/models/test_data_condition.py index ef61660252cde1..68c0d76d6737b1 100644 --- a/tests/sentry/workflow_engine/models/test_data_condition.py +++ b/tests/sentry/workflow_engine/models/test_data_condition.py @@ -2,10 +2,16 @@ import pytest +from sentry.seer.anomaly_detection.types import ( + AnomalyDetectionSeasonality, + AnomalyDetectionSensitivity, + AnomalyDetectionThresholdType, +) from sentry.testutils.cases import TestCase from sentry.workflow_engine.models.data_condition import Condition, DataConditionEvaluationException -from sentry.workflow_engine.types import DetectorPriorityLevel from tests.sentry.workflow_engine.test_base import DataConditionHandlerMixin +from sentry.workflow_engine.types import DataConditionHandler, DetectorPriorityLevel +from tests.sentry.workflow_engine.test_base import BaseWorkflowTest class GetConditionResultTest(TestCase): @@ -38,7 +44,7 @@ def test_boolean(self): assert dc.get_condition_result() is True -class EvaluateValueTest(DataConditionHandlerMixin, TestCase): +class EvaluateValueTest(DataConditionHandlerMixin, BaseWorkflowTest): def test(self): dc = self.create_data_condition( type=Condition.GREATER, comparison=1.0, condition_result=DetectorPriorityLevel.HIGH @@ -46,6 +52,21 @@ def test(self): assert dc.evaluate_value(2) == DetectorPriorityLevel.HIGH assert dc.evaluate_value(1) is None + def test_non_bool_result(self): + _, _, _, self.workflow_triggers = self.create_detector_and_workflow() + dc = self.create_data_condition( + type=Condition.ANOMALY_DETECTION, + comparison={ + "sensitivity": AnomalyDetectionSensitivity.MEDIUM, + "seasonality": AnomalyDetectionSeasonality.AUTO, + "threshold_type": AnomalyDetectionThresholdType.ABOVE_AND_BELOW, + }, + condition_result=DetectorPriorityLevel.HIGH, + condition_group=self.workflow_triggers, + ) + assert dc.evaluate_value(2) == DetectorPriorityLevel.HIGH + assert dc.evaluate_value(0) == DetectorPriorityLevel.OK + def test_bad_condition(self): with pytest.raises(ValueError): # Raises ValueError because the condition is invalid From 753968eb0191b4ebfae2611e035b2560ea4d2e69 Mon Sep 17 00:00:00 2001 From: Colleen O'Rourke Date: Wed, 28 May 2025 14:21:44 -0700 Subject: [PATCH 03/11] typing --- .../handlers/condition/anomaly_detection_handler.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sentry/workflow_engine/handlers/condition/anomaly_detection_handler.py b/src/sentry/workflow_engine/handlers/condition/anomaly_detection_handler.py index 3de0f14c4384e0..1ee1e28b37fd7f 100644 --- a/src/sentry/workflow_engine/handlers/condition/anomaly_detection_handler.py +++ b/src/sentry/workflow_engine/handlers/condition/anomaly_detection_handler.py @@ -37,5 +37,6 @@ class AnomalyDetectionHandler(DataConditionHandler[WorkflowEventData]): @staticmethod def evaluate_value(event_data: WorkflowEventData, comparison: int) -> DetectorPriorityLevel: - # this is a placeholder - return DetectorPriorityLevel.HIGH if event_data > 1 else DetectorPriorityLevel.OK + # this is a placeholder, type does not matter for now + value: int = event_data # type: ignore[assignment] + return DetectorPriorityLevel.HIGH if value > 1 else DetectorPriorityLevel.OK From 6dc276cc53147346b1846753304ef4556de3c812 Mon Sep 17 00:00:00 2001 From: Colleen O'Rourke Date: Thu, 29 May 2025 15:56:21 -0700 Subject: [PATCH 04/11] manually add to registry --- .../models/test_data_condition.py | 74 +++++++++++++++---- 1 file changed, 59 insertions(+), 15 deletions(-) diff --git a/tests/sentry/workflow_engine/models/test_data_condition.py b/tests/sentry/workflow_engine/models/test_data_condition.py index 68c0d76d6737b1..25c9a254c34ec9 100644 --- a/tests/sentry/workflow_engine/models/test_data_condition.py +++ b/tests/sentry/workflow_engine/models/test_data_condition.py @@ -10,10 +10,40 @@ from sentry.testutils.cases import TestCase from sentry.workflow_engine.models.data_condition import Condition, DataConditionEvaluationException from tests.sentry.workflow_engine.test_base import DataConditionHandlerMixin -from sentry.workflow_engine.types import DataConditionHandler, DetectorPriorityLevel +from sentry.workflow_engine.types import DataConditionHandler, DetectorPriorityLevel, WorkflowEventData +from tests.sentry.workflow_engine.endpoints.validators.test_base_data_condition import ( + MockDataConditionEnum, + MockDataConditionHandlerDictComparison, +) +from sentry.workflow_engine.registry import condition_handler_registry from tests.sentry.workflow_engine.test_base import BaseWorkflowTest +class MockDataConditionEnum(IntEnum): + FOO = 1 + BAR = 2 + + +class MockDataConditionHandlerDictComparison(DataConditionHandler): + group = DataConditionHandler.Group.DETECTOR_TRIGGER + comparison_json_schema = { + "type": "object", + "properties": { + "baz": {"type": "integer", "enum": [*MockDataConditionEnum]}, + }, + "required": ["baz"], + "additionalProperties": False, + } + + @staticmethod + def evaluate_value( + event_data: WorkflowEventData, comparison: dict[str, MockDataConditionEnum] + ) -> DetectorPriorityLevel: + return ( + DetectorPriorityLevel.HIGH if comparison["baz"].value > 1 else DetectorPriorityLevel.OK + ) + + class GetConditionResultTest(TestCase): def test_str(self): dc = self.create_data_condition(condition_result="wrong") @@ -45,27 +75,41 @@ def test_boolean(self): class EvaluateValueTest(DataConditionHandlerMixin, BaseWorkflowTest): - def test(self): - dc = self.create_data_condition( - type=Condition.GREATER, comparison=1.0, condition_result=DetectorPriorityLevel.HIGH + def setUp(self): + super().setUp() + self.workflow_triggers = self.create_data_condition_group() + self.dict_comparison_dc = self.create_data_condition( + type="test", + comparison={ + "baz": MockDataConditionEnum.FOO, + } ) - assert dc.evaluate_value(2) == DetectorPriorityLevel.HIGH - assert dc.evaluate_value(1) is None - - def test_non_bool_result(self): - _, _, _, self.workflow_triggers = self.create_detector_and_workflow() - dc = self.create_data_condition( - type=Condition.ANOMALY_DETECTION, + condition_handler_registry.registrations[Condition.REAPPEARED_EVENT] = ( + MockDataConditionHandlerDictComparison + ) + self.workflow_triggers = self.create_data_condition_group() + self.dict_comparison_dc = self.create_data_condition( + type=Condition.REAPPEARED_EVENT, comparison={ - "sensitivity": AnomalyDetectionSensitivity.MEDIUM, - "seasonality": AnomalyDetectionSeasonality.AUTO, - "threshold_type": AnomalyDetectionThresholdType.ABOVE_AND_BELOW, + "baz": MockDataConditionEnum.BAR, }, condition_result=DetectorPriorityLevel.HIGH, condition_group=self.workflow_triggers, ) + + def test(self): + dc = self.create_data_condition( + type=Condition.GREATER, comparison=1.0, condition_result=DetectorPriorityLevel.HIGH + ) assert dc.evaluate_value(2) == DetectorPriorityLevel.HIGH - assert dc.evaluate_value(0) == DetectorPriorityLevel.OK + assert dc.evaluate_value(1) is None + + def test_dict_comparison_result_high(self): + assert self.dict_comparison_dc.evaluate_value(2) == DetectorPriorityLevel.HIGH + + def test_dict_comparison_result_ok(self): + self.dict_comparison_dc.update(comparison={"baz": MockDataConditionEnum.FOO}) + assert self.dict_comparison_dc.evaluate_value(0) == DetectorPriorityLevel.OK def test_bad_condition(self): with pytest.raises(ValueError): From 801a6482ce94cbc93fd3eb8dd0f5d556ee672635 Mon Sep 17 00:00:00 2001 From: Colleen O'Rourke Date: Thu, 29 May 2025 16:14:11 -0700 Subject: [PATCH 05/11] ignore typing --- tests/sentry/workflow_engine/models/test_data_condition.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/sentry/workflow_engine/models/test_data_condition.py b/tests/sentry/workflow_engine/models/test_data_condition.py index 25c9a254c34ec9..238dd011ac5ac3 100644 --- a/tests/sentry/workflow_engine/models/test_data_condition.py +++ b/tests/sentry/workflow_engine/models/test_data_condition.py @@ -86,7 +86,7 @@ def setUp(self): ) condition_handler_registry.registrations[Condition.REAPPEARED_EVENT] = ( MockDataConditionHandlerDictComparison - ) + ) # type:ignore[assignment] self.workflow_triggers = self.create_data_condition_group() self.dict_comparison_dc = self.create_data_condition( type=Condition.REAPPEARED_EVENT, From 22e575ad7aecd27b9df8fd67cd1dd30604ffad4d Mon Sep 17 00:00:00 2001 From: Colleen O'Rourke Date: Mon, 2 Jun 2025 12:00:01 -0700 Subject: [PATCH 06/11] rebase --- .../models/test_data_condition.py | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/tests/sentry/workflow_engine/models/test_data_condition.py b/tests/sentry/workflow_engine/models/test_data_condition.py index 238dd011ac5ac3..37e54a16fca62d 100644 --- a/tests/sentry/workflow_engine/models/test_data_condition.py +++ b/tests/sentry/workflow_engine/models/test_data_condition.py @@ -1,3 +1,5 @@ +from enum import IntEnum +from typing import Any, cast from unittest import mock import pytest @@ -10,12 +12,12 @@ from sentry.testutils.cases import TestCase from sentry.workflow_engine.models.data_condition import Condition, DataConditionEvaluationException from tests.sentry.workflow_engine.test_base import DataConditionHandlerMixin -from sentry.workflow_engine.types import DataConditionHandler, DetectorPriorityLevel, WorkflowEventData -from tests.sentry.workflow_engine.endpoints.validators.test_base_data_condition import ( - MockDataConditionEnum, - MockDataConditionHandlerDictComparison, -) from sentry.workflow_engine.registry import condition_handler_registry +from sentry.workflow_engine.types import ( + DataConditionHandler, + DetectorPriorityLevel, + WorkflowEventData, +) from tests.sentry.workflow_engine.test_base import BaseWorkflowTest @@ -75,18 +77,11 @@ def test_boolean(self): class EvaluateValueTest(DataConditionHandlerMixin, BaseWorkflowTest): - def setUp(self): + def setUp(self) -> None: super().setUp() - self.workflow_triggers = self.create_data_condition_group() - self.dict_comparison_dc = self.create_data_condition( - type="test", - comparison={ - "baz": MockDataConditionEnum.FOO, - } + condition_handler_registry.registrations[Condition.REAPPEARED_EVENT] = cast( + DataConditionHandler[Any], MockDataConditionHandlerDictComparison ) - condition_handler_registry.registrations[Condition.REAPPEARED_EVENT] = ( - MockDataConditionHandlerDictComparison - ) # type:ignore[assignment] self.workflow_triggers = self.create_data_condition_group() self.dict_comparison_dc = self.create_data_condition( type=Condition.REAPPEARED_EVENT, From 5a47707bc23f391425fafa173c6f737b9c225f54 Mon Sep 17 00:00:00 2001 From: Colleen O'Rourke Date: Mon, 2 Jun 2025 12:01:48 -0700 Subject: [PATCH 07/11] weird rebase --- .../test_anomaly_detection_handler.py | 173 ------------------ 1 file changed, 173 deletions(-) delete mode 100644 tests/sentry/incidents/handlers/condition/test_anomaly_detection_handler.py diff --git a/tests/sentry/incidents/handlers/condition/test_anomaly_detection_handler.py b/tests/sentry/incidents/handlers/condition/test_anomaly_detection_handler.py deleted file mode 100644 index cd21dbd9e57bef..00000000000000 --- a/tests/sentry/incidents/handlers/condition/test_anomaly_detection_handler.py +++ /dev/null @@ -1,173 +0,0 @@ -from datetime import UTC, datetime -from unittest import mock - -import orjson -import pytest -from urllib3.response import HTTPResponse - -from sentry.incidents.handlers.condition.anomaly_detection_handler import DetectorError -from sentry.incidents.utils.types import MetricDetectorUpdate -from sentry.seer.anomaly_detection.types import ( - AnomalyDetectionSeasonality, - AnomalyDetectionSensitivity, - AnomalyDetectionThresholdType, - AnomalyType, - DataSourceType, - DetectAnomaliesResponse, -) -from sentry.snuba.subscriptions import create_snuba_subscription -from sentry.workflow_engine.models import Condition, DataPacket -from sentry.workflow_engine.types import DetectorPriorityLevel -from tests.sentry.workflow_engine.handlers.condition.test_base import ConditionTestCase - - -class TestAnomalyDetectionHandler(ConditionTestCase): - condition = Condition.ANOMALY_DETECTION - - def setUp(self): - super().setUp() - self.snuba_query = self.create_snuba_query() - self.subscription = create_snuba_subscription(self.project, "test", self.snuba_query) - - (self.workflow, self.detector, self.detector_workflow, self.workflow_triggers) = ( - self.create_detector_and_workflow() - ) - - subscription_update: MetricDetectorUpdate = { - "subscription_id": str(self.subscription.id), - "values": {"value": 1}, - "timestamp": datetime.now(UTC), - "entity": "test-entity", - } - - self.data_source = self.create_data_source( - source_id=str(subscription_update["subscription_id"]), - organization=self.organization, - ) - self.data_source.detectors.add(self.detector) - - self.data_packet = DataPacket[MetricDetectorUpdate]( - source_id=str(subscription_update["subscription_id"]), - packet=subscription_update, - ) - - self.dc = self.create_data_condition( - type=self.condition, - comparison={ - "sensitivity": AnomalyDetectionSensitivity.MEDIUM, - "seasonality": AnomalyDetectionSeasonality.AUTO, - "threshold_type": AnomalyDetectionThresholdType.ABOVE_AND_BELOW, - }, - condition_result=DetectorPriorityLevel.HIGH, - condition_group=self.workflow_triggers, - ) - - @mock.patch( - "sentry.seer.anomaly_detection.get_anomaly_data.SEER_ANOMALY_DETECTION_CONNECTION_POOL.urlopen" - ) - def test_passes(self, mock_seer_request): - seer_return_value: DetectAnomaliesResponse = { - "success": True, - "timeseries": [ - { - "anomaly": { - "anomaly_score": 0.9, - "anomaly_type": AnomalyType.HIGH_CONFIDENCE, - }, - "timestamp": 1, - "value": 10, - } - ], - } - mock_seer_request.return_value = HTTPResponse(orjson.dumps(seer_return_value), status=200) - self.assert_passes(self.dc, self.data_packet) - - @mock.patch( - "sentry.seer.anomaly_detection.get_anomaly_data.SEER_ANOMALY_DETECTION_CONNECTION_POOL.urlopen" - ) - def test_passes_medium(self, mock_seer_request): - seer_return_value: DetectAnomaliesResponse = { - "success": True, - "timeseries": [ - { - "anomaly": { - "anomaly_score": 0.2, - "anomaly_type": AnomalyType.LOW_CONFIDENCE, - }, - "timestamp": 1, - "value": 10, - } - ], - } - mock_seer_request.return_value = HTTPResponse(orjson.dumps(seer_return_value), status=200) - print("tset result: ", self.dc.evaluate_value(self.data_packet)) - assert self.dc.evaluate_value(self.data_packet) == DetectorPriorityLevel.OK.value - - @mock.patch( - "sentry.seer.anomaly_detection.get_anomaly_data.SEER_ANOMALY_DETECTION_CONNECTION_POOL.urlopen" - ) - @mock.patch("sentry.seer.anomaly_detection.get_anomaly_data.logger") - def test_seer_call_timeout_error(self, mock_logger, mock_seer_request): - from urllib3.exceptions import TimeoutError - - mock_seer_request.side_effect = TimeoutError - timeout_extra = { - "subscription_id": self.subscription.id, - "organization_id": self.organization.id, - "project_id": self.project.id, - "source_id": self.subscription.id, - "source_type": DataSourceType.SNUBA_QUERY_SUBSCRIPTION, - "dataset": self.subscription.snuba_query.dataset, - } - with pytest.raises(DetectorError): - self.dc.evaluate_value(self.data_packet) - mock_logger.warning.assert_called_with( - "Timeout error when hitting anomaly detection endpoint", extra=timeout_extra - ) - - @mock.patch( - "sentry.seer.anomaly_detection.get_anomaly_data.SEER_ANOMALY_DETECTION_CONNECTION_POOL.urlopen" - ) - @mock.patch("sentry.seer.anomaly_detection.get_anomaly_data.logger") - def test_seer_call_empty_list(self, mock_logger, mock_seer_request): - seer_return_value: DetectAnomaliesResponse = {"success": True, "timeseries": []} - mock_seer_request.return_value = HTTPResponse(orjson.dumps(seer_return_value), status=200) - with pytest.raises(DetectorError): - self.dc.evaluate_value(self.data_packet) - assert mock_logger.warning.call_args[0] == ( - "Seer anomaly detection response returned no potential anomalies", - ) - - @mock.patch( - "sentry.seer.anomaly_detection.get_anomaly_data.SEER_ANOMALY_DETECTION_CONNECTION_POOL.urlopen" - ) - @mock.patch("sentry.seer.anomaly_detection.get_anomaly_data.logger") - def test_seer_call_bad_status(self, mock_logger, mock_seer_request): - mock_seer_request.return_value = HTTPResponse(status=403) - extra = { - "subscription_id": self.subscription.id, - "organization_id": self.organization.id, - "project_id": self.project.id, - "source_id": self.subscription.id, - "source_type": DataSourceType.SNUBA_QUERY_SUBSCRIPTION, - "dataset": self.subscription.snuba_query.dataset, - "response_data": None, - } - with pytest.raises(DetectorError): - self.dc.evaluate_value(self.data_packet) - mock_logger.error.assert_called_with( - "Error when hitting Seer detect anomalies endpoint", extra=extra - ) - - @mock.patch( - "sentry.seer.anomaly_detection.get_anomaly_data.SEER_ANOMALY_DETECTION_CONNECTION_POOL.urlopen" - ) - @mock.patch("sentry.seer.anomaly_detection.get_anomaly_data.logger") - def test_seer_call_failed_parse(self, mock_logger, mock_seer_request): - # XXX: coercing a response into something that will fail to parse - mock_seer_request.return_value = HTTPResponse(None, status=200) # type: ignore[arg-type] - with pytest.raises(DetectorError): - self.dc.evaluate_value(self.data_packet) - mock_logger.exception.assert_called_with( - "Failed to parse Seer anomaly detection response", extra=mock.ANY - ) From 3d2e062bf4207bf503defb125b308ce5dc47ad13 Mon Sep 17 00:00:00 2001 From: Colleen O'Rourke Date: Mon, 2 Jun 2025 13:57:38 -0700 Subject: [PATCH 08/11] ugh rebase got weird again --- .../handlers/condition/__init__.py | 2 - .../condition/anomaly_detection_handler.py | 40 ++++--------------- .../models/test_data_condition.py | 8 +--- 3 files changed, 8 insertions(+), 42 deletions(-) diff --git a/src/sentry/workflow_engine/handlers/condition/__init__.py b/src/sentry/workflow_engine/handlers/condition/__init__.py index 2b4d0d53b66487..5f79c87f30d0d9 100644 --- a/src/sentry/workflow_engine/handlers/condition/__init__.py +++ b/src/sentry/workflow_engine/handlers/condition/__init__.py @@ -1,6 +1,5 @@ __all__ = [ "AgeComparisonConditionHandler", - "AnomalyDetectionHandler", "AssignedToConditionHandler", "EventAttributeConditionHandler", "EventCreatedByDetectorConditionHandler", @@ -27,7 +26,6 @@ ] from .age_comparison_handler import AgeComparisonConditionHandler -from .anomaly_detection_handler import AnomalyDetectionHandler from .assigned_to_handler import AssignedToConditionHandler from .event_attribute_handler import EventAttributeConditionHandler from .event_created_by_detector_handler import EventCreatedByDetectorConditionHandler diff --git a/src/sentry/workflow_engine/handlers/condition/anomaly_detection_handler.py b/src/sentry/workflow_engine/handlers/condition/anomaly_detection_handler.py index 1ee1e28b37fd7f..16dd1ddb6e4430 100644 --- a/src/sentry/workflow_engine/handlers/condition/anomaly_detection_handler.py +++ b/src/sentry/workflow_engine/handlers/condition/anomaly_detection_handler.py @@ -1,42 +1,16 @@ -from sentry.seer.anomaly_detection.types import ( - AnomalyDetectionSeasonality, - AnomalyDetectionSensitivity, - AnomalyDetectionThresholdType, -) +from typing import Any + from sentry.workflow_engine.models.data_condition import Condition from sentry.workflow_engine.registry import condition_handler_registry -from sentry.workflow_engine.types import ( - DataConditionHandler, - DetectorPriorityLevel, - WorkflowEventData, -) +from sentry.workflow_engine.types import DataConditionHandler, WorkflowEventData @condition_handler_registry.register(Condition.ANOMALY_DETECTION) class AnomalyDetectionHandler(DataConditionHandler[WorkflowEventData]): group = DataConditionHandler.Group.DETECTOR_TRIGGER - comparison_json_schema = { - "type": "object", - "properties": { - "sensitivity": { - "type": "string", - "enum": [*AnomalyDetectionSensitivity], - }, - "seasonality": { - "type": "string", - "enum": [*AnomalyDetectionSeasonality], - }, - "threshold_type": { - "type": "integer", - "enum": [*AnomalyDetectionThresholdType], - }, - }, - "required": ["sensitivity", "seasonality", "threshold_type"], - "additionalProperties": False, - } + comparison_json_schema = {"type": "boolean"} @staticmethod - def evaluate_value(event_data: WorkflowEventData, comparison: int) -> DetectorPriorityLevel: - # this is a placeholder, type does not matter for now - value: int = event_data # type: ignore[assignment] - return DetectorPriorityLevel.HIGH if value > 1 else DetectorPriorityLevel.OK + def evaluate_value(event_data: WorkflowEventData, comparison: Any) -> bool: + # this is a placeholder + return False diff --git a/tests/sentry/workflow_engine/models/test_data_condition.py b/tests/sentry/workflow_engine/models/test_data_condition.py index 37e54a16fca62d..f100867409e95e 100644 --- a/tests/sentry/workflow_engine/models/test_data_condition.py +++ b/tests/sentry/workflow_engine/models/test_data_condition.py @@ -4,21 +4,15 @@ import pytest -from sentry.seer.anomaly_detection.types import ( - AnomalyDetectionSeasonality, - AnomalyDetectionSensitivity, - AnomalyDetectionThresholdType, -) from sentry.testutils.cases import TestCase from sentry.workflow_engine.models.data_condition import Condition, DataConditionEvaluationException -from tests.sentry.workflow_engine.test_base import DataConditionHandlerMixin from sentry.workflow_engine.registry import condition_handler_registry from sentry.workflow_engine.types import ( DataConditionHandler, DetectorPriorityLevel, WorkflowEventData, ) -from tests.sentry.workflow_engine.test_base import BaseWorkflowTest +from tests.sentry.workflow_engine.test_base import BaseWorkflowTest, DataConditionHandlerMixin class MockDataConditionEnum(IntEnum): From 83ff42066f68598d32171e1c297a86e4b93ff492 Mon Sep 17 00:00:00 2001 From: Colleen O'Rourke Date: Mon, 2 Jun 2025 13:58:49 -0700 Subject: [PATCH 09/11] Remove that --- src/sentry/seer/anomaly_detection/types.py | 29 +--------------------- 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/src/sentry/seer/anomaly_detection/types.py b/src/sentry/seer/anomaly_detection/types.py index 48bba49da8dd59..af45ffef4b65a0 100644 --- a/src/sentry/seer/anomaly_detection/types.py +++ b/src/sentry/seer/anomaly_detection/types.py @@ -1,4 +1,4 @@ -from enum import Enum, IntEnum, StrEnum +from enum import Enum from typing import NotRequired, TypedDict @@ -74,30 +74,3 @@ class AnomalyType(Enum): LOW_CONFIDENCE = "anomaly_lower_confidence" NONE = "none" NO_DATA = "no_data" - - -class AnomalyDetectionSensitivity(StrEnum): - LOW = "low" - MEDIUM = "medium" - HIGH = "high" - - -class AnomalyDetectionSeasonality(StrEnum): - """All combinations of multi select fields for anomaly detection alerts - We do not anticipate adding more - """ - - AUTO = "auto" - HOURLY = "hourly" - DAILY = "daily" - WEEKLY = "weekly" - HOURLY_DAILY = "hourly_daily" - HOURLY_WEEKLY = "hourly_weekly" - HOURLY_DAILY_WEEKLY = "hourly_daily_weekly" - DAILY_WEEKLY = "daily_weekly" - - -class AnomalyDetectionThresholdType(IntEnum): - ABOVE = 0 - BELOW = 1 - ABOVE_AND_BELOW = 2 From 33604e9863f373c7610cf324ac675d8a73e283d4 Mon Sep 17 00:00:00 2001 From: Colleen O'Rourke Date: Mon, 2 Jun 2025 14:10:31 -0700 Subject: [PATCH 10/11] update test to use mixin --- .../models/test_data_condition.py | 64 +++++-------------- 1 file changed, 17 insertions(+), 47 deletions(-) diff --git a/tests/sentry/workflow_engine/models/test_data_condition.py b/tests/sentry/workflow_engine/models/test_data_condition.py index f100867409e95e..0ec830488dc566 100644 --- a/tests/sentry/workflow_engine/models/test_data_condition.py +++ b/tests/sentry/workflow_engine/models/test_data_condition.py @@ -1,17 +1,11 @@ from enum import IntEnum -from typing import Any, cast from unittest import mock import pytest from sentry.testutils.cases import TestCase from sentry.workflow_engine.models.data_condition import Condition, DataConditionEvaluationException -from sentry.workflow_engine.registry import condition_handler_registry -from sentry.workflow_engine.types import ( - DataConditionHandler, - DetectorPriorityLevel, - WorkflowEventData, -) +from sentry.workflow_engine.types import DetectorPriorityLevel from tests.sentry.workflow_engine.test_base import BaseWorkflowTest, DataConditionHandlerMixin @@ -20,26 +14,6 @@ class MockDataConditionEnum(IntEnum): BAR = 2 -class MockDataConditionHandlerDictComparison(DataConditionHandler): - group = DataConditionHandler.Group.DETECTOR_TRIGGER - comparison_json_schema = { - "type": "object", - "properties": { - "baz": {"type": "integer", "enum": [*MockDataConditionEnum]}, - }, - "required": ["baz"], - "additionalProperties": False, - } - - @staticmethod - def evaluate_value( - event_data: WorkflowEventData, comparison: dict[str, MockDataConditionEnum] - ) -> DetectorPriorityLevel: - return ( - DetectorPriorityLevel.HIGH if comparison["baz"].value > 1 else DetectorPriorityLevel.OK - ) - - class GetConditionResultTest(TestCase): def test_str(self): dc = self.create_data_condition(condition_result="wrong") @@ -71,21 +45,6 @@ def test_boolean(self): class EvaluateValueTest(DataConditionHandlerMixin, BaseWorkflowTest): - def setUp(self) -> None: - super().setUp() - condition_handler_registry.registrations[Condition.REAPPEARED_EVENT] = cast( - DataConditionHandler[Any], MockDataConditionHandlerDictComparison - ) - self.workflow_triggers = self.create_data_condition_group() - self.dict_comparison_dc = self.create_data_condition( - type=Condition.REAPPEARED_EVENT, - comparison={ - "baz": MockDataConditionEnum.BAR, - }, - condition_result=DetectorPriorityLevel.HIGH, - condition_group=self.workflow_triggers, - ) - def test(self): dc = self.create_data_condition( type=Condition.GREATER, comparison=1.0, condition_result=DetectorPriorityLevel.HIGH @@ -93,12 +52,23 @@ def test(self): assert dc.evaluate_value(2) == DetectorPriorityLevel.HIGH assert dc.evaluate_value(1) is None - def test_dict_comparison_result_high(self): - assert self.dict_comparison_dc.evaluate_value(2) == DetectorPriorityLevel.HIGH + def test_dict_comparison_result(self): + def evaluate_value(value: int, comparison: int) -> bool: + return ( + DetectorPriorityLevel.HIGH + if comparison["baz"].value > 1 + else DetectorPriorityLevel.OK + ) + + dc = self.setup_condition_mocks( + evaluate_value, ["sentry.workflow_engine.models.data_condition"] + ) + dc.update(comparison={"baz": MockDataConditionEnum.BAR}) + assert dc.evaluate_value(2) == DetectorPriorityLevel.HIGH - def test_dict_comparison_result_ok(self): - self.dict_comparison_dc.update(comparison={"baz": MockDataConditionEnum.FOO}) - assert self.dict_comparison_dc.evaluate_value(0) == DetectorPriorityLevel.OK + dc.update(comparison={"baz": MockDataConditionEnum.FOO}) + assert dc.evaluate_value(0) == DetectorPriorityLevel.OK + self.teardown_condition_mocks() def test_bad_condition(self): with pytest.raises(ValueError): From 0b92d2fa383b8a2370baa54e220691c8c20703a5 Mon Sep 17 00:00:00 2001 From: Colleen O'Rourke Date: Mon, 2 Jun 2025 14:56:15 -0700 Subject: [PATCH 11/11] typing --- tests/sentry/workflow_engine/models/test_data_condition.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/sentry/workflow_engine/models/test_data_condition.py b/tests/sentry/workflow_engine/models/test_data_condition.py index 0ec830488dc566..c059bcb2d5da27 100644 --- a/tests/sentry/workflow_engine/models/test_data_condition.py +++ b/tests/sentry/workflow_engine/models/test_data_condition.py @@ -53,7 +53,9 @@ def test(self): assert dc.evaluate_value(1) is None def test_dict_comparison_result(self): - def evaluate_value(value: int, comparison: int) -> bool: + def evaluate_value( + value: int, comparison: dict[str, DetectorPriorityLevel] + ) -> DetectorPriorityLevel: return ( DetectorPriorityLevel.HIGH if comparison["baz"].value > 1