Adding schema collection to sqlserver

sqlserver/assets/configuration/spec.yaml

@@ -713,6 +713,30 @@ files:
         type: number
         example: 1800
         display_default: false
+    - name: schemas_collection 
+      description: |
+        Configure collection of schemas. If `database_autodiscovery` is not enabled, data is collected 
+        only for the database configured with `database` parameter.
+      options:
+        - name: enabled
+          description: |
+            Enable schema collection. Requires `dbm: true`. Defaults to false.
+          value:
+            type: boolean
+            example: false
+        - name: collection_interval
+          description: |
+            Set the database schema collection interval (in seconds). Defaults to 600 seconds.
+          value:
+            type: number
+            example: 600
+        - name: max_execution_time
+          description: |
+            Set the maximum time for schema collection (in seconds). Defaults to 10 seconds. 
+            Capped by `schemas_collection.collection_interval`
+          value:
+            type: number
+            example: 10
     - template: instances/default
   - template: logs
     example:

sqlserver/changelog.d/17258.added

@@ -0,0 +1,2 @@
+Adding schema collection to sqlserver
+Schema data includes information about the tables, their columns, indexes, foreign keys, and partitions.

sqlserver/datadog_checks/sqlserver/config.py

@@ -7,7 +7,10 @@
 
 from datadog_checks.base.config import is_affirmative
 from datadog_checks.base.utils.common import to_native_string
-from datadog_checks.sqlserver.const import DEFAULT_AUTODISCOVERY_INTERVAL, PROC_CHAR_LIMIT
+from datadog_checks.sqlserver.const import (
+    DEFAULT_AUTODISCOVERY_INTERVAL,
+    PROC_CHAR_LIMIT,
+)
 
 
 class SQLServerConfig:
@@ -45,6 +48,7 @@ def __init__(self, init_config, instance, log):
         self.procedure_metrics_config: dict = instance.get('procedure_metrics', {}) or {}
         self.settings_config: dict = instance.get('collect_settings', {}) or {}
         self.activity_config: dict = instance.get('query_activity', {}) or {}
+        self.schema_config: dict = instance.get('schemas_collection', {}) or {}
         self.cloud_metadata: dict = {}
         aws: dict = instance.get('aws', {}) or {}
         gcp: dict = instance.get('gcp', {}) or {}

sqlserver/datadog_checks/sqlserver/config_models/instance.py

@@ -139,6 +139,16 @@ class QueryMetrics(BaseModel):
     samples_per_hour_per_query: Optional[int] = None
 
 
+class SchemasCollection(BaseModel):
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        frozen=True,
+    )
+    collection_interval: Optional[float] = None
+    enabled: Optional[bool] = None
+    max_execution_time: Optional[float] = None
+
+
 class InstanceConfig(BaseModel):
     model_config = ConfigDict(
         validate_default=True,
@@ -199,6 +209,7 @@ class InstanceConfig(BaseModel):
     query_activity: Optional[QueryActivity] = None
     query_metrics: Optional[QueryMetrics] = None
     reported_hostname: Optional[str] = None
+    schemas_collection: Optional[SchemasCollection] = None
     server_version: Optional[str] = None
     service: Optional[str] = None
     stored_procedure: Optional[str] = None

sqlserver/datadog_checks/sqlserver/const.py

@@ -268,3 +268,5 @@
 ]
 
 PROC_CHAR_LIMIT = 500
+
+DEFAULT_SCHEMAS_COLLECTION_INTERVAL = 600

sqlserver/datadog_checks/sqlserver/data/conf.yaml.example

@@ -659,6 +659,27 @@ instances:
     #
     # ignore_missing_database: false
 
+    ## Configure collection of schemas. If `database_autodiscovery` is not enabled, data is collected 
+    ## only for the database configured with `database` parameter.
+    #
+    # schemas_collection:
+
+        ## @param enabled - boolean - optional - default: false
+        ## Enable schema collection. Requires `dbm: true`. Defaults to false.
+        #
+        # enabled: false
+
+        ## @param collection_interval - number - optional - default: 600
+        ## Set the database schema collection interval (in seconds). Defaults to 600 seconds.
+        #
+        # collection_interval: 600
+
+        ## @param max_execution_time - number - optional - default: 10
+        ## Set the maximum time for schema collection (in seconds). Defaults to 10 seconds. 
+        ## Capped by `schemas_collection.collection_interval`
+        #
+        # max_execution_time: 10
+
     ## @param tags - list of strings - optional
     ## A list of tags to attach to every metric and service check emitted by this instance.
     ##

sqlserver/datadog_checks/sqlserver/queries.py

@@ -143,6 +143,73 @@
     ],
 }
 
+DB_QUERY = """
+SELECT
+    db.database_id AS id, db.name AS name, db.collation_name AS collation, dp.name AS owner
+FROM
+    sys.databases db LEFT JOIN sys.database_principals dp ON db.owner_sid = dp.sid
+WHERE db.name IN ({});
+"""
+
+SCHEMA_QUERY = """
+SELECT
+    s.name AS name, s.schema_id AS id, dp.name AS owner_name
+FROM
+    sys.schemas AS s JOIN sys.database_principals dp ON s.principal_id = dp.principal_id
+WHERE s.name NOT IN ('sys', 'information_schema')
+"""
+
+TABLES_IN_SCHEMA_QUERY = """
+SELECT
+    object_id AS id, name
+FROM
+    sys.tables
+WHERE schema_id=?
+"""
+
+COLUMN_QUERY = """
+SELECT
+    column_name AS name, data_type, column_default, is_nullable AS nullable , table_name, ordinal_position
+FROM
+    information_schema.columns
+WHERE
+    table_name IN ({}) and table_schema='{}';
+"""
+
+PARTITIONS_QUERY = """
+SELECT
+    object_id AS id, COUNT(*) AS partition_count
+FROM
+    sys.partitions
+WHERE
+    object_id IN ({}) GROUP BY object_id;
+"""
+
+INDEX_QUERY = """
+SELECT
+    i.object_id AS id, i.name, i.type, i.is_unique, i.is_primary_key, i.is_unique_constraint,
+    i.is_disabled, STRING_AGG(c.name, ',') AS column_names
+FROM
+    sys.indexes i JOIN sys.index_columns ic ON i.object_id = ic.object_id
+    AND i.index_id = ic.index_id JOIN sys.columns c ON ic.object_id = c.object_id AND ic.column_id = c.column_id
+WHERE
+    i.object_id IN ({}) GROUP BY i.object_id, i.name, i.type,
+    i.is_unique, i.is_primary_key, i.is_unique_constraint, i.is_disabled;
+"""
+
+FOREIGN_KEY_QUERY = """
+SELECT
+    FK.referenced_object_id AS id, FK.name AS foreign_key_name,
+    OBJECT_NAME(FK.parent_object_id) AS referencing_table,
+    STRING_AGG(COL_NAME(FKC.parent_object_id, FKC.parent_column_id),',') AS referencing_column,
+    OBJECT_NAME(FK.referenced_object_id) AS referenced_table,
+    STRING_AGG(COL_NAME(FKC.referenced_object_id, FKC.referenced_column_id),',') AS referenced_column
+FROM
+    sys.foreign_keys AS FK JOIN sys.foreign_key_columns AS FKC ON FK.object_id = FKC.constraint_object_id
+WHERE
+    FK.referenced_object_id IN ({}) GROUP BY FK.name, FK.parent_object_id, FK.referenced_object_id;
+"""
+
 
 def get_query_ao_availability_groups(sqlserver_major_version):
     """