New database instance identifier for Postges, MySQL, SQLServer (#19341) (#20088)

* Add unique database identifiers

* Lint

* WIP

* WIP

* WIP

* WIP

* Fix

* Postgres identifier

* WIP

* WIP

* Test fix

* Mysql/SQLServer

* Sort tags

* Clean

* Changelog

* Fix PR numbers

* Lint

* Sync

* WIP

* Tests

* Tests

* WIP

* WIP

* WIP

* WIP

* MySQL

* Lint

* Get

* Tag filter

* Fix E2E

* WIP

* WIP

* Tests

* Lint

* Docs

(cherry picked from commit 76c4247)

Co-authored-by: Seth Samuel <seth.samuel@datadoghq.com>

Author: datadog-agent-integrations-bot[bot]

mysql/assets/configuration/spec.yaml

@@ -70,7 +70,29 @@ files:
               and can be useful to set a custom hostname when connecting to a remote database through a proxy.
             value:
               type: string
+          - name: database_identifier
+            description: |
+              Controls how the database is identified. The default value is the resolved hostname for the instance, 
+              which respects the `reported_hostname` option.
 
+              This value will be used as-is for the display name of the instance but will be normalized
+              when applied as the `database_instance` tag. Please see https://docs.datadoghq.com/getting_started/tagging/
+              for more details on Datadog tag normalization.
+            options:
+              - name: template
+                value:
+                  type: string
+                  display_default: "$resolved_hostname"
+                  example: "$env-$resolved_hostname:$port"
+                description: |
+                  The template to use for the database identifier. The default value is `$resolved_hostname`.
+                  You can use the following variables, prefixed by `$` in the template:
+                  - resolved_hostname: The resolved hostname of the instance, which respects the `reported_hostname` option.
+                  - host: The provided host of the instance.
+                  - port: The port number of the instance.
+                  - mysql_sock: The socket path of the instance.
+                  In addition, you can use any key from the `tags` section of the configuration.
+                default: "$resolved_hostname"
           - name: sock
             description: |
               Path to a Unix Domain Socket to use when connecting to MySQL (instead of a TCP socket).

mysql/changelog.d/19341.added

@@ -0,0 +1,2 @@
+Added a new configuration option `database_identifier.template`. Use this template to specify the unique identifier for a database instance, separate from the underlying host.
+The `empty_default_hostname` configuration option is now respected and will omit the `host` tag from database instances when enabled.

mysql/datadog_checks/mysql/activity.py

@@ -195,7 +195,7 @@ def run_job(self):
                         'https://docs.datadoghq.com/database_monitoring/setup_mysql/troubleshooting#%s',
                         DatabaseConfigurationError.events_waits_current_not_enabled.value,
                         code=DatabaseConfigurationError.events_waits_current_not_enabled.value,
-                        host=self._check.resolved_hostname,
+                        host=self._check.reported_hostname,
                     ),
                 )
             return
@@ -332,7 +332,7 @@ def _get_estimated_row_size_bytes(row):
     def _create_activity_event(self, active_sessions, tags):
         # type: (List[Dict[str]], List[Dict[str]]) -> Dict[str]
         return {
-            "host": self._check.resolved_hostname,
+            "host": self._check.reported_hostname,
             "ddagentversion": datadog_agent.get_version(),
             "ddsource": "mysql",
             "dbm_type": "activity",

mysql/datadog_checks/mysql/config.py

@@ -12,6 +12,8 @@
 class MySQLConfig(object):
     def __init__(self, instance, init_config):
         self.log = get_check_logger()
+        self.database_identifier = instance.get('database_identifier', {})
+        self.empty_default_hostname = instance.get("empty_default_hostname", False)
         self.host = instance.get('host', instance.get('server', ''))
         self.port = int(instance.get('port', 0))
         self.reported_hostname = instance.get('reported_hostname', '')

mysql/datadog_checks/mysql/config_models/instance.py

@@ -58,6 +58,14 @@ class CustomQuery(BaseModel):
     tags: Optional[tuple[str, ...]] = None
 
 
+class DatabaseIdentifier(BaseModel):
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        frozen=True,
+    )
+    template: Optional[str] = None
+
+
 class Gcp(BaseModel):
     model_config = ConfigDict(
         arbitrary_types_allowed=True,
@@ -200,6 +208,7 @@ class InstanceConfig(BaseModel):
     collect_settings: Optional[CollectSettings] = None
     connect_timeout: Optional[float] = None
     custom_queries: Optional[tuple[CustomQuery, ...]] = None
+    database_identifier: Optional[DatabaseIdentifier] = None
     database_instance_collection_interval: Optional[float] = None
     dbm: Optional[bool] = None
     defaults_file: Optional[str] = None

mysql/datadog_checks/mysql/data/conf.yaml.example

@@ -64,6 +64,26 @@ instances:
     #
     # reported_hostname: <REPORTED_HOSTNAME>
 
+    ## Controls how the database is identified. The default value is the resolved hostname for the instance, 
+    ## which respects the `reported_hostname` option.
+    ##
+    ## This value will be used as-is for the display name of the instance but will be normalized
+    ## when applied as the `database_instance` tag. Please see https://docs.datadoghq.com/getting_started/tagging/
+    ## for more details on Datadog tag normalization.
+    #
+    # database_identifier:
+
+        ## @param template - string - optional - default: $resolved_hostname
+        ## The template to use for the database identifier. The default value is `$resolved_hostname`.
+        ## You can use the following variables, prefixed by `$` in the template:
+        ## - resolved_hostname: The resolved hostname of the instance, which respects the `reported_hostname` option.
+        ## - host: The provided host of the instance.
+        ## - port: The port number of the instance.
+        ## - mysql_sock: The socket path of the instance.
+        ## In addition, you can use any key from the `tags` section of the configuration.
+        #
+        # template: $env-$resolved_hostname:$port
+
     ## @param sock - string - optional
     ## Path to a Unix Domain Socket to use when connecting to MySQL (instead of a TCP socket).
     ## If you specify a socket you dont need to specify a host.

mysql/datadog_checks/mysql/databases_data.py

@@ -157,7 +157,7 @@ def _cursor_run(self, cursor, query, params=None):
                 "dd.mysql.db.error",
                 1,
                 tags=self._tags + ["error:{}".format(type(e))] + self._check._get_debug_tags(),
-                hostname=self._check.resolved_hostname,
+                hostname=self._check.reported_hostname,
             )
             raise
 

mysql/datadog_checks/mysql/metadata.py

@@ -118,7 +118,7 @@ def _cursor_run(self, cursor, query, params=None):
                 "dd.mysql.db.error",
                 1,
                 tags=self._tags + ["error:{}".format(type(e))] + self._check._get_debug_tags(),
-                hostname=self._check.resolved_hostname,
+                hostname=self._check.reported_hostname,
             )
             raise
 
@@ -169,7 +169,7 @@ def report_mysql_metadata(self):
             rows = cursor.fetchall()
             settings = [dict(row) for row in rows]
         event = {
-            "host": self._check.resolved_hostname,
+            "host": self._check.reported_hostname,
             "agent_version": datadog_agent.get_version(),
             "dbms": "mysql",
             "kind": "mysql_variables",

mysql/datadog_checks/mysql/mysql.py

@@ -9,6 +9,7 @@
 import traceback
 from collections import defaultdict
 from contextlib import closing, contextmanager
+from string import Template
 from typing import Any, Dict, List, Optional  # noqa: F401
 
 import pymysql
@@ -109,11 +110,13 @@ def __init__(self, name, init_config, instances):
         self.version = None
         self.is_mariadb = None
         self._resolved_hostname = None
+        self._database_identifier = None
         self._agent_hostname = None
         self._database_hostname = None
         self._is_aurora = None
         self._config = MySQLConfig(self.instance, init_config)
         self.tags = self._config.tags
+        self.add_core_tags()
         self.cloud_metadata = self._config.cloud_metadata
 
         # Create a new connection on every check run
@@ -157,15 +160,46 @@ def _send_metadata(self):
         self.set_metadata('flavor', self.version.flavor)
         self.set_metadata('resolved_hostname', self.resolved_hostname)
 
+    @property
+    def reported_hostname(self):
+        # type: () -> str
+        if self._config.empty_default_hostname:
+            return None
+        return self.resolved_hostname
+
     @property
     def resolved_hostname(self):
+        # type: () -> str
         if self._resolved_hostname is None:
             if self._config.reported_hostname:
                 self._resolved_hostname = self._config.reported_hostname
             else:
                 self._resolved_hostname = self.resolve_db_host()
         return self._resolved_hostname
 
+    @property
+    def database_identifier(self):
+        # type: () -> str
+        if self._database_identifier is None:
+            template = Template(self._config.database_identifier.get('template') or '$resolved_hostname')
+            tag_dict = {}
+            tags = self.tags.copy()
+            # sort tags to ensure consistent ordering
+            tags.sort()
+            for t in tags:
+                if ':' in t:
+                    key, value = t.split(':', 1)
+                    if key in tag_dict:
+                        tag_dict[key] += f",{value}"
+                    else:
+                        tag_dict[key] = value
+            tag_dict['resolved_hostname'] = self.resolved_hostname
+            tag_dict['host'] = str(self._config.host)
+            tag_dict['port'] = str(self._config.port)
+            tag_dict['mysql_sock'] = str(self._config.mysql_sock)
+            self._database_identifier = template.safe_substitute(**tag_dict)
+        return self._database_identifier
+
     @property
     def agent_hostname(self):
         # type: () -> str
@@ -180,9 +214,14 @@ def database_hostname(self):
             self._database_hostname = self.resolve_db_host()
         return self._database_hostname
 
-    def set_resource_tags(self):
+    def add_core_tags(self):
+        """
+        Add tags that should be attached to every metric/event but which require check calculations outside the config.
+        """
         self.tags.append("database_hostname:{}".format(self.database_hostname))
+        self.tags.append("database_instance:{}".format(self.database_identifier))
 
+    def set_resource_tags(self):
         if self.cloud_metadata.get("gcp") is not None:
             self.tags.append(
                 "dd.internal.resource:gcp_sql_database_instance:{}:{}".format(
@@ -199,6 +238,9 @@ def set_resource_tags(self):
             # allow for detecting if the host is an RDS host, and emit
             # the resource properly even if the `aws` config is unset
             self.tags.append("dd.internal.resource:aws_rds_instance:{}".format(self.resolved_hostname))
+            self.cloud_metadata["aws"] = {
+                "instance_endpoint": self.resolved_hostname,
+            }
         if self.cloud_metadata.get("azure") is not None:
             deployment_type = self.cloud_metadata.get("azure")["deployment_type"]
             # some `deployment_type`s map to multiple `resource_type`s
@@ -210,7 +252,7 @@ def set_resource_tags(self):
         # finally, emit a `database_instance` resource for this instance
         self.tags.append(
             "dd.internal.resource:database_instance:{}".format(
-                self.resolved_hostname,
+                self.database_identifier,
             )
         )
 
@@ -371,7 +413,7 @@ def _new_query_executor(self, queries):
             self.execute_query_raw,
             self,
             queries=queries,
-            hostname=self.resolved_hostname,
+            hostname=self.reported_hostname,
             track_operation_time=True,
         )
 
@@ -467,12 +509,12 @@ def _connect(self):
             self.log.debug("Connected to MySQL")
             self.service_check_tags = list(set(service_check_tags))
             self.service_check(
-                self.SERVICE_CHECK_NAME, AgentCheck.OK, tags=service_check_tags, hostname=self.resolved_hostname
+                self.SERVICE_CHECK_NAME, AgentCheck.OK, tags=service_check_tags, hostname=self.reported_hostname
             )
             yield db
         except Exception:
             self.service_check(
-                self.SERVICE_CHECK_NAME, AgentCheck.CRITICAL, tags=service_check_tags, hostname=self.resolved_hostname
+                self.SERVICE_CHECK_NAME, AgentCheck.CRITICAL, tags=service_check_tags, hostname=self.reported_hostname
             )
             raise
         finally:
@@ -796,20 +838,20 @@ def _submit_replication_status(self, status, additional_tags):
             name=self.SLAVE_SERVICE_CHECK_NAME,
             value=1 if status == AgentCheck.OK else 0,
             tags=self.tags + additional_tags,
-            hostname=self.resolved_hostname,
+            hostname=self.reported_hostname,
         )
         # deprecated in favor of service_check("mysql.replication.replica_running")
         self.service_check(
             self.SLAVE_SERVICE_CHECK_NAME,
             status,
             tags=self.service_check_tags + additional_tags,
-            hostname=self.resolved_hostname,
+            hostname=self.reported_hostname,
         )
         self.service_check(
             self.REPLICA_SERVICE_CHECK_NAME,
             status,
             tags=self.service_check_tags + additional_tags,
-            hostname=self.resolved_hostname,
+            hostname=self.reported_hostname,
         )
 
     def _is_source_host(self, replicas, results):
@@ -858,13 +900,13 @@ def __submit_metric(self, metric_name, metric_type, variable, db_results, tags):
                     metric_tags.append(tag)
             if value is not None:
                 if metric_type == RATE:
-                    self.rate(metric_name, value, tags=metric_tags, hostname=self.resolved_hostname)
+                    self.rate(metric_name, value, tags=metric_tags, hostname=self.reported_hostname)
                 elif metric_type == GAUGE:
-                    self.gauge(metric_name, value, tags=metric_tags, hostname=self.resolved_hostname)
+                    self.gauge(metric_name, value, tags=metric_tags, hostname=self.reported_hostname)
                 elif metric_type == COUNT:
-                    self.count(metric_name, value, tags=metric_tags, hostname=self.resolved_hostname)
+                    self.count(metric_name, value, tags=metric_tags, hostname=self.reported_hostname)
                 elif metric_type == MONOTONIC:
-                    self.monotonic_count(metric_name, value, tags=metric_tags, hostname=self.resolved_hostname)
+                    self.monotonic_count(metric_name, value, tags=metric_tags, hostname=self.reported_hostname)
 
     def _collect_dict(self, metric_type, field_metric_map, query, db, tags):
         """
@@ -891,15 +933,15 @@ def _collect_dict(self, metric_type, field_metric_map, query, db, tags):
                                 self.log.debug("Collecting done, value %s", result[col_idx])
                                 if metric_type == GAUGE:
                                     self.gauge(
-                                        metric, float(result[col_idx]), tags=tags, hostname=self.resolved_hostname
+                                        metric, float(result[col_idx]), tags=tags, hostname=self.reported_hostname
                                     )
                                 elif metric_type == RATE:
                                     self.rate(
-                                        metric, float(result[col_idx]), tags=tags, hostname=self.resolved_hostname
+                                        metric, float(result[col_idx]), tags=tags, hostname=self.reported_hostname
                                     )
                                 else:
                                     self.gauge(
-                                        metric, float(result[col_idx]), tags=tags, hostname=self.resolved_hostname
+                                        metric, float(result[col_idx]), tags=tags, hostname=self.reported_hostname
                                     )
                             else:
                                 self.log.debug("Received value is None for index %d", col_idx)
@@ -956,10 +998,10 @@ def _collect_system_metrics(self, host, db, tags):
                     scpu = proc.cpu_times()[1]
 
                     if ucpu and scpu:
-                        self.rate("mysql.performance.user_time", ucpu, tags=tags, hostname=self.resolved_hostname)
+                        self.rate("mysql.performance.user_time", ucpu, tags=tags, hostname=self.reported_hostname)
                         # should really be system_time
-                        self.rate("mysql.performance.kernel_time", scpu, tags=tags, hostname=self.resolved_hostname)
-                        self.rate("mysql.performance.cpu_time", ucpu + scpu, tags=tags, hostname=self.resolved_hostname)
+                        self.rate("mysql.performance.kernel_time", scpu, tags=tags, hostname=self.reported_hostname)
+                        self.rate("mysql.performance.cpu_time", ucpu + scpu, tags=tags, hostname=self.reported_hostname)
                 else:
                     self.log.debug("psutil is not available, will not collect mysql.performance.* metrics")
             except Exception:
@@ -1334,10 +1376,11 @@ def _report_warnings(self):
             self.warning(warning)
 
     def _send_database_instance_metadata(self):
-        if self.resolved_hostname not in self._database_instance_emitted:
+        if self.database_identifier not in self._database_instance_emitted:
             event = {
-                "host": self.resolved_hostname,
+                "host": self.reported_hostname,
                 "port": self._config.port,
+                "database_instance": self.database_identifier,
                 "database_hostname": self.database_hostname,
                 "agent_version": datadog_agent.get_version(),
                 "dbms": "mysql",
@@ -1353,5 +1396,5 @@ def _send_database_instance_metadata(self):
                     "connection_host": self._config.host,
                 },
             }
-            self._database_instance_emitted[self.resolved_hostname] = event
+            self._database_instance_emitted[self.database_identifier] = event
             self.database_monitoring_metadata(json.dumps(event, default=default_json_event_encoding))