Skip to content

Commit 280db3e

Browse files
JoshFergeMichaelSun48
authored andcommitted
feat(feedback): spam detection (#69169)
- Adds llm based spam detection field to issue evidence if the algorithm returns that the message is spam. behind a feature flag.
1 parent bd7c8a6 commit 280db3e

File tree

4 files changed

+166
-4
lines changed

4 files changed

+166
-4
lines changed

src/sentry/feedback/usecases/create_feedback.py

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,10 @@
88

99
import jsonschema
1010

11+
from sentry import features
1112
from sentry.constants import DataCategory
1213
from sentry.eventstore.models import Event
14+
from sentry.feedback.usecases.spam_detection import is_spam
1315
from sentry.issues.grouptype import FeedbackGroup
1416
from sentry.issues.issue_occurrence import IssueEvidence, IssueOccurrence
1517
from sentry.issues.json_schemas import EVENT_PAYLOAD_SCHEMA, LEGACY_EVENT_PAYLOAD_SCHEMA
@@ -54,7 +56,7 @@ def old_feedback_category_values(cls) -> set[str]:
5456
}
5557

5658

57-
def make_evidence(feedback, source: FeedbackCreationSource):
59+
def make_evidence(feedback, source: FeedbackCreationSource, is_message_spam: bool | None):
5860
evidence_data = {}
5961
evidence_display = []
6062
if feedback.get("contact_email"):
@@ -74,6 +76,12 @@ def make_evidence(feedback, source: FeedbackCreationSource):
7476
evidence_data["source"] = source.value
7577
evidence_display.append(IssueEvidence(name="source", value=source.value, important=False))
7678

79+
if is_message_spam is True:
80+
evidence_data["is_spam"] = str(is_message_spam)
81+
evidence_display.append(
82+
IssueEvidence(name="is_spam", value=str(is_message_spam), important=False)
83+
)
84+
7785
return evidence_data, evidence_display
7886

7987

@@ -169,11 +177,23 @@ def create_feedback_issue(event, project_id, source: FeedbackCreationSource):
169177
if should_filter_feedback(event, project_id, source):
170178
return
171179

180+
project = Project.objects.get_from_cache(id=project_id)
181+
182+
is_message_spam = None
183+
if features.has("organizations:user-feedback-spam-filter-ingest", project.organization):
184+
try:
185+
is_message_spam = is_spam(event["contexts"]["feedback"]["message"])
186+
except Exception:
187+
# until we have LLM error types ironed out, just catch all exceptions
188+
logger.exception("Error checking if message is spam")
189+
172190
# Note that some of the fields below like title and subtitle
173191
# are not used by the feedback UI, but are required.
174192
event["event_id"] = event.get("event_id") or uuid4().hex
175193
detection_time = datetime.fromtimestamp(event["timestamp"], UTC)
176-
evidence_data, evidence_display = make_evidence(event["contexts"]["feedback"], source)
194+
evidence_data, evidence_display = make_evidence(
195+
event["contexts"]["feedback"], source, is_message_spam
196+
)
177197
occurrence = IssueOccurrence(
178198
id=uuid4().hex,
179199
event_id=event.get("event_id") or uuid4().hex,
@@ -204,8 +224,6 @@ def create_feedback_issue(event, project_id, source: FeedbackCreationSource):
204224
# make sure event data is valid for issue platform
205225
validate_issue_platform_event_schema(event_fixed)
206226

207-
project = Project.objects.get_from_cache(id=project_id)
208-
209227
if not project.flags.has_feedbacks:
210228
first_feedback_received.send_robust(project=project, sender=Project)
211229

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
import logging
2+
3+
from sentry.llm.usecases import LLMUseCase, complete_prompt
4+
from sentry.utils import metrics
5+
6+
logger = logging.getLogger(__name__)
7+
8+
PROMPT = """Classify the text into one of the following two classes: [Junk, Not Junk]. Choose Junk only if you are confident. Text: """
9+
10+
11+
@metrics.wraps("feedback.spam_detection", sample_rate=1.0)
12+
def is_spam(message):
13+
is_spam = False
14+
response = complete_prompt(usecase=LLMUseCase.SPAM_DETECTION, prompt=PROMPT, message=message)
15+
if response and response.lower() == "junk":
16+
is_spam = True
17+
18+
logger.info(
19+
"Spam detection",
20+
extra={
21+
"feedback_message": message,
22+
"is_spam": is_spam,
23+
"response": response,
24+
},
25+
)
26+
metrics.incr("spam-detection", tags={"is_spam": is_spam}, sample_rate=1.0)
27+
return is_spam

src/sentry/llm/usecases/__init__.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
class LLMUseCase(Enum):
1919
EXAMPLE = "example" # used in tests / examples
2020
SUGGESTED_FIX = "suggestedfix" # OG version of suggested fix
21+
SPAM_DETECTION = "spamdetection"
2122

2223

2324
llm_provider_backends: dict[str, LlmModelBase] = {}

tests/sentry/feedback/usecases/test_create_feedback.py

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,20 @@
11
from __future__ import annotations
22

3+
import time
34
from typing import Any
45
from unittest.mock import Mock
56

67
import pytest
8+
from openai.types.chat.chat_completion import ChatCompletion, Choice
9+
from openai.types.chat.chat_completion_message import ChatCompletionMessage
710

811
from sentry.feedback.usecases.create_feedback import (
912
FeedbackCreationSource,
1013
create_feedback_issue,
1114
fix_for_issue_platform,
1215
validate_issue_platform_event_schema,
1316
)
17+
from sentry.testutils.helpers import Feature
1418
from sentry.testutils.pytest.fixtures import django_db_all
1519

1620

@@ -23,6 +27,21 @@ def mock_produce_occurrence_to_kafka(monkeypatch):
2327
return mock
2428

2529

30+
@pytest.fixture(autouse=True)
31+
def llm_settings(set_sentry_option):
32+
with (
33+
set_sentry_option(
34+
"llm.provider.options",
35+
{"openai": {"models": ["gpt-4-turbo-1.0"], "options": {"api_key": "fake_api_key"}}},
36+
),
37+
set_sentry_option(
38+
"llm.usecases.options",
39+
{"spamdetection": {"provider": "openai", "options": {"model": "gpt-4-turbo-1.0"}}},
40+
),
41+
):
42+
yield
43+
44+
2645
def test_fix_for_issue_platform():
2746
event: dict[str, Any] = {
2847
"project_id": 1,
@@ -421,3 +440,100 @@ def test_create_feedback_filters_no_contexts_or_message(
421440
)
422441

423442
assert mock_produce_occurrence_to_kafka.call_count == 0
443+
444+
445+
@django_db_all
446+
@pytest.mark.parametrize(
447+
"input_message, expected_result, feature_flag",
448+
[
449+
("This is definitely spam", "True", True),
450+
("Valid feedback message", None, True),
451+
("This is definitely spam", None, False),
452+
("Valid feedback message", None, False),
453+
],
454+
)
455+
def test_create_feedback_spam_detection_adds_field(
456+
default_project,
457+
mock_produce_occurrence_to_kafka,
458+
input_message,
459+
expected_result,
460+
monkeypatch,
461+
feature_flag,
462+
):
463+
with Feature({"organizations:user-feedback-spam-filter-ingest": feature_flag}):
464+
event = {
465+
"project_id": default_project.id,
466+
"request": {
467+
"url": "https://sentry.sentry.io/feedback/?statsPeriod=14d",
468+
"headers": {
469+
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36"
470+
},
471+
},
472+
"event_id": "56b08cf7852c42cbb95e4a6998c66ad6",
473+
"timestamp": 1698255009.574,
474+
"received": "2021-10-24T22:23:29.574000+00:00",
475+
"environment": "prod",
476+
"release": "frontend@daf1316f209d961443664cd6eb4231ca154db502",
477+
"user": {
478+
"ip_address": "72.164.175.154",
479+
"email": "josh.ferge@sentry.io",
480+
"id": 880461,
481+
"isStaff": False,
482+
"name": "Josh Ferge",
483+
},
484+
"contexts": {
485+
"feedback": {
486+
"contact_email": "josh.ferge@sentry.io",
487+
"name": "Josh Ferge",
488+
"message": input_message,
489+
"replay_id": "3d621c61593c4ff9b43f8490a78ae18e",
490+
"url": "https://sentry.sentry.io/feedback/?statsPeriod=14d",
491+
},
492+
},
493+
"breadcrumbs": [],
494+
"platform": "javascript",
495+
}
496+
497+
def dummy_response(*args, **kwargs):
498+
return ChatCompletion(
499+
id="test",
500+
choices=[
501+
Choice(
502+
index=0,
503+
message=ChatCompletionMessage(
504+
content=(
505+
"Junk"
506+
if kwargs["messages"][1]["content"] == "This is definitely spam"
507+
else "Not Junk"
508+
),
509+
role="assistant",
510+
),
511+
finish_reason="stop",
512+
)
513+
],
514+
created=time.time(),
515+
model="gpt3.5-trubo",
516+
object="chat.completion",
517+
)
518+
519+
mock_openai = Mock()
520+
mock_openai().chat.completions.create = dummy_response
521+
522+
monkeypatch.setattr("sentry.llm.providers.openai.OpenAI", mock_openai)
523+
524+
create_feedback_issue(
525+
event, default_project.id, FeedbackCreationSource.NEW_FEEDBACK_ENVELOPE
526+
)
527+
528+
# Check if the 'is_spam' evidence in the Kafka message matches the expected result
529+
is_spam_evidence = [
530+
evidence.value
531+
for evidence in mock_produce_occurrence_to_kafka.call_args.kwargs[
532+
"occurrence"
533+
].evidence_display
534+
if evidence.name == "is_spam"
535+
]
536+
found_is_spam = is_spam_evidence[0] if is_spam_evidence else None
537+
assert (
538+
found_is_spam == expected_result
539+
), f"Expected {expected_result} but found {found_is_spam} for {input_message} and feature flag {feature_flag}"

0 commit comments

Comments
 (0)