From eea192d7cf55f4ca4c4479e3adddd1d10c1cec94 Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Thu, 15 May 2025 12:18:55 -0400 Subject: [PATCH 1/9] include object_name, not procedure name for rpc events --- .../sqlserver/xe_collection/query_completion_events.py | 3 +-- sqlserver/tests/test_xe_collection.py | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sqlserver/datadog_checks/sqlserver/xe_collection/query_completion_events.py b/sqlserver/datadog_checks/sqlserver/xe_collection/query_completion_events.py index c457d8e5131eb..6d8ae4eb0f449 100644 --- a/sqlserver/datadog_checks/sqlserver/xe_collection/query_completion_events.py +++ b/sqlserver/datadog_checks/sqlserver/xe_collection/query_completion_events.py @@ -45,12 +45,11 @@ class QueryCompletionEventsHandler(XESessionBase): "spills": 0, "row_count": 0, "object_id": 0, - "line_number": 0, } RPC_SPECIFIC_STRING_FIELDS = [ "result", - "procedure_name", + "object_name", "data_stream", "connection_reset_option", ] diff --git a/sqlserver/tests/test_xe_collection.py b/sqlserver/tests/test_xe_collection.py index 651203f2fe0e2..5fa5123827718 100644 --- a/sqlserver/tests/test_xe_collection.py +++ b/sqlserver/tests/test_xe_collection.py @@ -190,6 +190,7 @@ def rpc_completed_expected_values(): 'client_hostname': 'EC2AMAZ-ML3E0PH', 'client_app_name': 'SQLAgent - Job Manager', 'username': 'NT AUTHORITY\\NETWORK SERVICE', + 'object_name': 'sp_executesql', } @@ -621,6 +622,8 @@ def test_process_events_rpc_completed( assert 'sp_executesql' in event['statement'] assert 'sql_text' in event assert 'EXECUTE [msdb].[dbo].[sp_agent_log_job_history]' in event['sql_text'] + assert 'object_name' in event + assert event['object_name'] == 'sp_executesql' def test_process_events_error_reported(self, error_events_handler, sample_error_event_xml, error_expected_values): """Test processing of error reported events""" From 24668fe3d544ec81367d5186747e73abf8db9766 Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Thu, 15 May 2025 13:07:18 -0400 Subject: [PATCH 2/9] add changelog --- sqlserver/changelog.d/20303.fixed | 1 + 1 file changed, 1 insertion(+) create mode 100644 sqlserver/changelog.d/20303.fixed diff --git a/sqlserver/changelog.d/20303.fixed b/sqlserver/changelog.d/20303.fixed new file mode 100644 index 0000000000000..bf370a77dfe50 --- /dev/null +++ b/sqlserver/changelog.d/20303.fixed @@ -0,0 +1 @@ +include object_name, not procedure name for rpc events From d2888272e07138e5a8fb2df8d0b03ed39de96512 Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Thu, 15 May 2025 13:09:19 -0400 Subject: [PATCH 3/9] fix changelog --- sqlserver/changelog.d/{20303.fixed => 20302.fixed} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename sqlserver/changelog.d/{20303.fixed => 20302.fixed} (100%) diff --git a/sqlserver/changelog.d/20303.fixed b/sqlserver/changelog.d/20302.fixed similarity index 100% rename from sqlserver/changelog.d/20303.fixed rename to sqlserver/changelog.d/20302.fixed From 61f6f928139cbc70cd7584a143732885f892cd23 Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Thu, 15 May 2025 13:16:41 -0400 Subject: [PATCH 4/9] object id and spills also not real fields --- .../sqlserver/xe_collection/query_completion_events.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sqlserver/datadog_checks/sqlserver/xe_collection/query_completion_events.py b/sqlserver/datadog_checks/sqlserver/xe_collection/query_completion_events.py index 6d8ae4eb0f449..7bf02e89c9e0d 100644 --- a/sqlserver/datadog_checks/sqlserver/xe_collection/query_completion_events.py +++ b/sqlserver/datadog_checks/sqlserver/xe_collection/query_completion_events.py @@ -42,9 +42,7 @@ class QueryCompletionEventsHandler(XESessionBase): "physical_reads": 0, "logical_reads": 0, "writes": 0, - "spills": 0, "row_count": 0, - "object_id": 0, } RPC_SPECIFIC_STRING_FIELDS = [ From 4723f581e4359749c823d76485e76c8e945911de Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Thu, 15 May 2025 21:11:10 -0400 Subject: [PATCH 5/9] Include duration and query start for attention events --- sqlserver/changelog.d/20307.fixed | 1 + .../sqlserver/xe_collection/base.py | 20 ++- .../sqlserver/xe_collection/error_events.py | 11 +- sqlserver/tests/test_xe_collection.py | 141 ++++++++++++++++++ 4 files changed, 157 insertions(+), 16 deletions(-) create mode 100644 sqlserver/changelog.d/20307.fixed diff --git a/sqlserver/changelog.d/20307.fixed b/sqlserver/changelog.d/20307.fixed new file mode 100644 index 0000000000000..b28632901d7ab --- /dev/null +++ b/sqlserver/changelog.d/20307.fixed @@ -0,0 +1 @@ +Include duration and query start for attention events diff --git a/sqlserver/datadog_checks/sqlserver/xe_collection/base.py b/sqlserver/datadog_checks/sqlserver/xe_collection/base.py index 006ae25227a04..27b4abab019dc 100644 --- a/sqlserver/datadog_checks/sqlserver/xe_collection/base.py +++ b/sqlserver/datadog_checks/sqlserver/xe_collection/base.py @@ -731,23 +731,21 @@ def _create_rqt_event(self, event, raw_sql_fields, query_details): "event_fire_timestamp": query_details.get("event_fire_timestamp"), } - # Only include duration and query_start for non-error events - is_error_event = self.session_name == "datadog_query_errors" - if not is_error_event: + # Only exclude duration and query_start for error_reported events, not attention events + is_error_reported = event.get("event_name") == "error_reported" + if not is_error_reported: sqlserver_fields.update( { "duration_ms": event.get("duration_ms"), "query_start": query_details.get("query_start"), } ) - else: - # Include error_number and message for error events - sqlserver_fields.update( - { - "error_number": event.get("error_number"), - "message": event.get("message"), - } - ) + + # Include error_number and message if they're present in the event + if event.get("error_number") is not None: + sqlserver_fields["error_number"] = event.get("error_number") + if event.get("message"): + sqlserver_fields["message"] = event.get("message") # Add additional SQL fields to the sqlserver section # but only if they're not the primary field and not empty diff --git a/sqlserver/datadog_checks/sqlserver/xe_collection/error_events.py b/sqlserver/datadog_checks/sqlserver/xe_collection/error_events.py index e6271cbea2574..f20bc4b91c2c1 100644 --- a/sqlserver/datadog_checks/sqlserver/xe_collection/error_events.py +++ b/sqlserver/datadog_checks/sqlserver/xe_collection/error_events.py @@ -137,11 +137,12 @@ def _normalize_event_impl(self, event): # First use the base normalization with type-specific fields normalized = self._normalize_event(event) - # For error events, remove query_start and duration_ms fields since they're not applicable - if 'query_start' in normalized: - del normalized['query_start'] - if 'duration_ms' in normalized: - del normalized['duration_ms'] + # For error_reported events only, remove query_start and duration_ms fields since they're not applicable + if normalized.get('xe_type') == 'error_reported': + if 'query_start' in normalized: + del normalized['query_start'] + if 'duration_ms' in normalized: + del normalized['duration_ms'] return normalized diff --git a/sqlserver/tests/test_xe_collection.py b/sqlserver/tests/test_xe_collection.py index 5fa5123827718..906823ed31096 100644 --- a/sqlserver/tests/test_xe_collection.py +++ b/sqlserver/tests/test_xe_collection.py @@ -824,6 +824,34 @@ def test_normalize_error_event(self, error_events_handler): assert 'duration_ms' not in normalized assert 'query_start' not in normalized + def test_normalize_attention_event(self, error_events_handler): + """Test attention event normalization""" + # Test attention event with fields + event = { + 'event_name': 'attention', + 'timestamp': '2023-01-01T12:00:00.123Z', + 'duration_ms': 328.677, + 'session_id': 123, + 'request_id': 456, + 'database_name': 'TestDB', + 'sql_text': 'SELECT * FROM Customers WHERE CustomerId = 123', + } + + normalized = error_events_handler._normalize_event_impl(event) + + # Verify normalized fields + assert normalized['xe_type'] == 'attention' + assert normalized['event_fire_timestamp'] == '2023-01-01T12:00:00.123Z' + assert normalized['session_id'] == 123 + assert normalized['request_id'] == 456 + assert normalized['database_name'] == 'TestDB' + assert normalized['sql_text'] == 'SELECT * FROM Customers WHERE CustomerId = 123' + + # Verify duration_ms and query_start are preserved for attention events + assert 'duration_ms' in normalized + assert normalized['duration_ms'] == 328.677 + assert 'query_start' in normalized # Query start should be calculated from timestamp and duration + @patch('datadog_checks.sqlserver.xe_collection.base.datadog_agent') def test_create_event_payload(self, mock_agent, query_completion_handler): """Test creation of event payload""" @@ -902,6 +930,62 @@ def test_create_rqt_event(self, mock_agent, query_completion_handler): assert rqt_event['sqlserver']['duration_ms'] == 10.0 assert rqt_event['sqlserver']['query_start'] == '2023-01-01T11:59:50.123Z' + @patch('datadog_checks.sqlserver.xe_collection.base.datadog_agent') + def test_create_rqt_event_attention(self, mock_agent, error_events_handler): + """Test creation of Raw Query Text event for attention event""" + mock_agent.get_version.return_value = '7.30.0' + + # Create attention event with SQL fields - from the error_events_handler + event = { + 'event_name': 'attention', + 'timestamp': '2023-01-01T12:00:00.123Z', + 'duration_ms': 328.677, + 'session_id': 123, + 'database_name': 'TestDB', + 'sql_text': 'SELECT * FROM Customers WHERE CustomerId = ?', + 'query_signature': 'abc123', + 'primary_sql_field': 'sql_text', + 'dd_tables': ['Customers'], + 'dd_commands': ['SELECT'], + } + + # Create raw SQL fields + raw_sql_fields = { + 'sql_text': 'SELECT * FROM Customers WHERE CustomerId = 123', + 'raw_query_signature': 'def456', + } + + # Query details with formatted timestamps + query_details = { + 'event_fire_timestamp': '2023-01-01T12:00:00.123Z', + 'query_start': '2023-01-01T11:59:59.795Z', # 328.677ms before timestamp + 'duration_ms': 328.677, + } + + # Create RQT event + rqt_event = error_events_handler._create_rqt_event(event, raw_sql_fields, query_details) + + # Validate common payload fields + validate_common_payload_fields(rqt_event, expected_source='datadog_query_errors', expected_type='rqt') + + # Verify DB fields + assert rqt_event['db']['instance'] == 'TestDB' + assert rqt_event['db']['query_signature'] == 'abc123' + assert rqt_event['db']['raw_query_signature'] == 'def456' + assert rqt_event['db']['statement'] == 'SELECT * FROM Customers WHERE CustomerId = 123' + + # Verify sqlserver fields + assert rqt_event['sqlserver']['session_id'] == 123 + assert rqt_event['sqlserver']['xe_type'] == 'attention' + assert rqt_event['sqlserver']['event_fire_timestamp'] == '2023-01-01T12:00:00.123Z' + + # Key check: verify that duration_ms and query_start are present for attention events + # even though they come from the error_events_handler + assert 'duration_ms' in rqt_event['sqlserver'] + assert rqt_event['sqlserver']['duration_ms'] == 328.677 + assert 'query_start' in rqt_event['sqlserver'] + assert rqt_event['sqlserver']['query_start'] == '2023-01-01T11:59:59.795Z' + def test_create_rqt_event_disabled(self, mock_check, mock_config): """Test RQT event creation when disabled""" # Disable raw query collection @@ -948,6 +1032,63 @@ def test_create_rqt_event_missing_signature(self, query_completion_handler): # Should return None when missing signature assert query_completion_handler._create_rqt_event(event, raw_sql_fields, query_details) is None + @patch('datadog_checks.sqlserver.xe_collection.base.datadog_agent') + def test_create_rqt_event_error_reported(self, mock_agent, error_events_handler): + """Test creation of Raw Query Text event for error_reported event""" + mock_agent.get_version.return_value = '7.30.0' + + # Create error_reported event with SQL fields + event = { + 'event_name': 'error_reported', + 'timestamp': '2023-01-01T12:00:00.123Z', + 'error_number': 8134, + 'severity': 15, + 'session_id': 123, + 'database_name': 'TestDB', + 'sql_text': 'SELECT 1/0', + 'message': 'Division by zero error', + 'query_signature': 'abc123', + 'primary_sql_field': 'sql_text', + } + + # Create raw SQL fields + raw_sql_fields = { + 'sql_text': 'SELECT 1/0', + 'raw_query_signature': 'def456', + } + + # Query details would not have duration_ms or query_start for error_reported events + query_details = { + 'event_fire_timestamp': '2023-01-01T12:00:00.123Z', + } + + # Create RQT event + rqt_event = error_events_handler._create_rqt_event(event, raw_sql_fields, query_details) + + # Validate common payload fields + validate_common_payload_fields(rqt_event, expected_source='datadog_query_errors', expected_type='rqt') + + # Verify DB fields + assert rqt_event['db']['instance'] == 'TestDB' + assert rqt_event['db']['query_signature'] == 'abc123' + assert rqt_event['db']['raw_query_signature'] == 'def456' + assert rqt_event['db']['statement'] == 'SELECT 1/0' + + # Verify sqlserver fields + assert rqt_event['sqlserver']['session_id'] == 123 + assert rqt_event['sqlserver']['xe_type'] == 'error_reported' + assert rqt_event['sqlserver']['event_fire_timestamp'] == '2023-01-01T12:00:00.123Z' + + # Key check: verify that error_number and message are included for error_reported events + assert 'error_number' in rqt_event['sqlserver'] + assert rqt_event['sqlserver']['error_number'] == 8134 + assert 'message' in rqt_event['sqlserver'] + assert rqt_event['sqlserver']['message'] == 'Division by zero error' + + # Verify that duration_ms and query_start are NOT present for error_reported events + assert 'duration_ms' not in rqt_event['sqlserver'] + assert 'query_start' not in rqt_event['sqlserver'] + @pytest.mark.integration @pytest.mark.usefixtures('dd_environment') From b4ab497a0143ea3043389eda1fa85512bd704e9d Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Thu, 15 May 2025 21:36:32 -0400 Subject: [PATCH 6/9] activity id and activity id xfer --- .../sqlserver/xe_collection/error_events.py | 15 +++++++++++++-- .../xe_collection/query_completion_events.py | 3 ++- sqlserver/tests/test_xe_collection.py | 4 ++++ sqlserver/tests/xml_xe_events/attention.xml | 12 ++++++++++++ sqlserver/tests/xml_xe_events/error_reported.xml | 12 ++++++++++++ 5 files changed, 43 insertions(+), 3 deletions(-) diff --git a/sqlserver/datadog_checks/sqlserver/xe_collection/error_events.py b/sqlserver/datadog_checks/sqlserver/xe_collection/error_events.py index f20bc4b91c2c1..3f5a450dd9367 100644 --- a/sqlserver/datadog_checks/sqlserver/xe_collection/error_events.py +++ b/sqlserver/datadog_checks/sqlserver/xe_collection/error_events.py @@ -93,7 +93,14 @@ def _process_error_reported_event(self, event, event_data): # Extract action elements for action in event.findall('./action'): action_name = action.get('name') - if action_name: + if not action_name: + continue + + if action_name == 'attach_activity_id': + event_data['activity_id'] = extract_value(action) + elif action_name == 'attach_activity_id_xfer': + event_data['activity_id_xfer'] = extract_value(action) + else: event_data[action_name] = extract_value(action) return True @@ -122,7 +129,11 @@ def _process_attention_event(self, event, event_data): if not action_name: continue - if action_name == 'session_id' or action_name == 'request_id': + if action_name == 'attach_activity_id': + event_data['activity_id'] = extract_value(action) + elif action_name == 'attach_activity_id_xfer': + event_data['activity_id_xfer'] = extract_value(action) + elif action_name == 'session_id' or action_name == 'request_id': # These are numeric values in the actions value = extract_int_value(action) if value is not None: diff --git a/sqlserver/datadog_checks/sqlserver/xe_collection/query_completion_events.py b/sqlserver/datadog_checks/sqlserver/xe_collection/query_completion_events.py index 7bf02e89c9e0d..9de8c08e63c83 100644 --- a/sqlserver/datadog_checks/sqlserver/xe_collection/query_completion_events.py +++ b/sqlserver/datadog_checks/sqlserver/xe_collection/query_completion_events.py @@ -143,9 +143,10 @@ def _process_action_elements(self, event, event_data): for action in event.findall('./action'): action_name = action.get('name') if action_name: - # Add activity_id support if action_name == 'attach_activity_id': event_data['activity_id'] = extract_value(action) + elif action_name == 'attach_activity_id_xfer': + event_data['activity_id_xfer'] = extract_value(action) else: event_data[action_name] = extract_value(action) diff --git a/sqlserver/tests/test_xe_collection.py b/sqlserver/tests/test_xe_collection.py index 906823ed31096..af666de705cbe 100644 --- a/sqlserver/tests/test_xe_collection.py +++ b/sqlserver/tests/test_xe_collection.py @@ -209,6 +209,8 @@ def error_expected_values(): 'client_app_name': 'go-mssqldb', 'username': 'shopper_4', 'message': "'REPEAT' is not a recognized built-in function name.", + 'activity_id': 'F961B15C-752A-487E-AC4F-F2A9BAB11DB7-1', + 'activity_id_xfer': 'AFCCDE6F-EACD-47F3-9B62-CC02D517191B-0', } @@ -254,6 +256,8 @@ def attention_expected_values(): 'client_hostname': 'COMP-MX2YQD7P2P', 'client_app_name': 'azdata', 'username': 'datadog', + 'activity_id': 'F961B15C-752A-487E-AC4F-F2A9BAB11DB7-1', + 'activity_id_xfer': 'AFCCDE6F-EACD-47F3-9B62-CC02D517191B-0', } diff --git a/sqlserver/tests/xml_xe_events/attention.xml b/sqlserver/tests/xml_xe_events/attention.xml index 400cd0ff9daf5..eced01c290daa 100644 --- a/sqlserver/tests/xml_xe_events/attention.xml +++ b/sqlserver/tests/xml_xe_events/attention.xml @@ -90,4 +90,16 @@ ORDER BY event_timestamp; + + + + AFCCDE6F-EACD-47F3-9B62-CC02D517191B-0 + + + + + + F961B15C-752A-487E-AC4F-F2A9BAB11DB7-1 + + diff --git a/sqlserver/tests/xml_xe_events/error_reported.xml b/sqlserver/tests/xml_xe_events/error_reported.xml index 884bed388b8bd..3820d1c63a36d 100644 --- a/sqlserver/tests/xml_xe_events/error_reported.xml +++ b/sqlserver/tests/xml_xe_events/error_reported.xml @@ -95,4 +95,16 @@ /*dddbs='orders-app',ddps='orders-app',ddh='awbergs-sqlserver2019-test.c7ug0vvtkhqv.us-east-1.rds.amazonaws.com',dddb='dbmorders',ddprs='orders-sqlserver'*/ SELECT discount_percent, store_name, description, discount_in_currency, dbm_item_id, REPEAT('a', 1000) from discount where id BETWEEN 6117 AND 6127 GROUP by dbm_item_id, store_name, description, discount_in_currency, discount_percent /* date='12%2F31',key='val' */ + + + + AFCCDE6F-EACD-47F3-9B62-CC02D517191B-0 + + + + + + F961B15C-752A-487E-AC4F-F2A9BAB11DB7-1 + + From cbbcbab4183d208d2054389f327e56a9582b5072 Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Thu, 15 May 2025 21:40:27 -0400 Subject: [PATCH 7/9] add activity_id_xfer to base string fields --- sqlserver/datadog_checks/sqlserver/xe_collection/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sqlserver/datadog_checks/sqlserver/xe_collection/base.py b/sqlserver/datadog_checks/sqlserver/xe_collection/base.py index 27b4abab019dc..1769939295760 100644 --- a/sqlserver/datadog_checks/sqlserver/xe_collection/base.py +++ b/sqlserver/datadog_checks/sqlserver/xe_collection/base.py @@ -95,6 +95,7 @@ class XESessionBase(DBMAsyncJob): "client_app_name", "username", "activity_id", + "activity_id_xfer", ] BASE_SQL_FIELDS = [ From 2a077c591c8442b4fe44f2d19a15c79e02b493d5 Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Thu, 15 May 2025 22:31:25 -0400 Subject: [PATCH 8/9] changelogs --- sqlserver/changelog.d/20302.fixed | 2 +- sqlserver/changelog.d/20307.fixed | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 sqlserver/changelog.d/20307.fixed diff --git a/sqlserver/changelog.d/20302.fixed b/sqlserver/changelog.d/20302.fixed index bf370a77dfe50..c40280022c125 100644 --- a/sqlserver/changelog.d/20302.fixed +++ b/sqlserver/changelog.d/20302.fixed @@ -1 +1 @@ -include object_name, not procedure name for rpc events +Fill in missing XE events \ No newline at end of file diff --git a/sqlserver/changelog.d/20307.fixed b/sqlserver/changelog.d/20307.fixed deleted file mode 100644 index b28632901d7ab..0000000000000 --- a/sqlserver/changelog.d/20307.fixed +++ /dev/null @@ -1 +0,0 @@ -Include duration and query start for attention events From 564701fb0467f5251b9216e49ce72eaf602536ee Mon Sep 17 00:00:00 2001 From: Allen Zhou Date: Fri, 16 May 2025 13:15:21 -0400 Subject: [PATCH 9/9] Update 20302.fixed --- sqlserver/changelog.d/20302.fixed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqlserver/changelog.d/20302.fixed b/sqlserver/changelog.d/20302.fixed index c40280022c125..287d3eb75f9a9 100644 --- a/sqlserver/changelog.d/20302.fixed +++ b/sqlserver/changelog.d/20302.fixed @@ -1 +1 @@ -Fill in missing XE events \ No newline at end of file +Add object_name to rpc_completed XE events, add activity ID and activity ID Xfer to error events