1
1
from __future__ import annotations
2
2
3
+ import time
3
4
from typing import Any
4
5
from unittest .mock import Mock
5
6
6
7
import pytest
8
+ from openai .types .chat .chat_completion import ChatCompletion , Choice
9
+ from openai .types .chat .chat_completion_message import ChatCompletionMessage
7
10
8
11
from sentry .feedback .usecases .create_feedback import (
9
12
FeedbackCreationSource ,
10
13
create_feedback_issue ,
11
14
fix_for_issue_platform ,
12
15
validate_issue_platform_event_schema ,
13
16
)
17
+ from sentry .testutils .helpers import Feature
14
18
from sentry .testutils .pytest .fixtures import django_db_all
15
19
16
20
@@ -23,6 +27,21 @@ def mock_produce_occurrence_to_kafka(monkeypatch):
23
27
return mock
24
28
25
29
30
+ @pytest .fixture (autouse = True )
31
+ def llm_settings (set_sentry_option ):
32
+ with (
33
+ set_sentry_option (
34
+ "llm.provider.options" ,
35
+ {"openai" : {"models" : ["gpt-4-turbo-1.0" ], "options" : {"api_key" : "fake_api_key" }}},
36
+ ),
37
+ set_sentry_option (
38
+ "llm.usecases.options" ,
39
+ {"spamdetection" : {"provider" : "openai" , "options" : {"model" : "gpt-4-turbo-1.0" }}},
40
+ ),
41
+ ):
42
+ yield
43
+
44
+
26
45
def test_fix_for_issue_platform ():
27
46
event : dict [str , Any ] = {
28
47
"project_id" : 1 ,
@@ -421,3 +440,100 @@ def test_create_feedback_filters_no_contexts_or_message(
421
440
)
422
441
423
442
assert mock_produce_occurrence_to_kafka .call_count == 0
443
+
444
+
445
+ @django_db_all
446
+ @pytest .mark .parametrize (
447
+ "input_message, expected_result, feature_flag" ,
448
+ [
449
+ ("This is definitely spam" , "True" , True ),
450
+ ("Valid feedback message" , None , True ),
451
+ ("This is definitely spam" , None , False ),
452
+ ("Valid feedback message" , None , False ),
453
+ ],
454
+ )
455
+ def test_create_feedback_spam_detection_adds_field (
456
+ default_project ,
457
+ mock_produce_occurrence_to_kafka ,
458
+ input_message ,
459
+ expected_result ,
460
+ monkeypatch ,
461
+ feature_flag ,
462
+ ):
463
+ with Feature ({"organizations:user-feedback-spam-filter-ingest" : feature_flag }):
464
+ event = {
465
+ "project_id" : default_project .id ,
466
+ "request" : {
467
+ "url" : "https://sentry.sentry.io/feedback/?statsPeriod=14d" ,
468
+ "headers" : {
469
+ "User-Agent" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36"
470
+ },
471
+ },
472
+ "event_id" : "56b08cf7852c42cbb95e4a6998c66ad6" ,
473
+ "timestamp" : 1698255009.574 ,
474
+ "received" : "2021-10-24T22:23:29.574000+00:00" ,
475
+ "environment" : "prod" ,
476
+ "release" : "frontend@daf1316f209d961443664cd6eb4231ca154db502" ,
477
+ "user" : {
478
+ "ip_address" : "72.164.175.154" ,
479
+ "email" : "josh.ferge@sentry.io" ,
480
+ "id" : 880461 ,
481
+ "isStaff" : False ,
482
+ "name" : "Josh Ferge" ,
483
+ },
484
+ "contexts" : {
485
+ "feedback" : {
486
+ "contact_email" : "josh.ferge@sentry.io" ,
487
+ "name" : "Josh Ferge" ,
488
+ "message" : input_message ,
489
+ "replay_id" : "3d621c61593c4ff9b43f8490a78ae18e" ,
490
+ "url" : "https://sentry.sentry.io/feedback/?statsPeriod=14d" ,
491
+ },
492
+ },
493
+ "breadcrumbs" : [],
494
+ "platform" : "javascript" ,
495
+ }
496
+
497
+ def dummy_response (* args , ** kwargs ):
498
+ return ChatCompletion (
499
+ id = "test" ,
500
+ choices = [
501
+ Choice (
502
+ index = 0 ,
503
+ message = ChatCompletionMessage (
504
+ content = (
505
+ "Junk"
506
+ if kwargs ["messages" ][1 ]["content" ] == "This is definitely spam"
507
+ else "Not Junk"
508
+ ),
509
+ role = "assistant" ,
510
+ ),
511
+ finish_reason = "stop" ,
512
+ )
513
+ ],
514
+ created = time .time (),
515
+ model = "gpt3.5-trubo" ,
516
+ object = "chat.completion" ,
517
+ )
518
+
519
+ mock_openai = Mock ()
520
+ mock_openai ().chat .completions .create = dummy_response
521
+
522
+ monkeypatch .setattr ("sentry.llm.providers.openai.OpenAI" , mock_openai )
523
+
524
+ create_feedback_issue (
525
+ event , default_project .id , FeedbackCreationSource .NEW_FEEDBACK_ENVELOPE
526
+ )
527
+
528
+ # Check if the 'is_spam' evidence in the Kafka message matches the expected result
529
+ is_spam_evidence = [
530
+ evidence .value
531
+ for evidence in mock_produce_occurrence_to_kafka .call_args .kwargs [
532
+ "occurrence"
533
+ ].evidence_display
534
+ if evidence .name == "is_spam"
535
+ ]
536
+ found_is_spam = is_spam_evidence [0 ] if is_spam_evidence else None
537
+ assert (
538
+ found_is_spam == expected_result
539
+ ), f"Expected { expected_result } but found { found_is_spam } for { input_message } and feature flag { feature_flag } "
0 commit comments