Adding schema collection to sqlserver (#17258)

* Adding schema collection to sqlserver

* rather use sys tables

* snapshot collect data

* remove unused function

* small improvments

* improving code

* fixed errors

* refactored code

* Introduced a function that iterates between databases

* minor changes

* put in a separate class

* some clean up

* Corrected column query

* added partitions count

* Added foreign count

* Added stop

* fixed errors

* fixed errors

* removed old code

* Fixed some bugs in chunk schema collection

* removed some comments

* some diffs

* working version send data in chunks

* introduced collection per tables

* Introduced a class for data submit

* pretending to be postgres for testing

* Adopted to Postgres

* adopted payload to the backend

* remove breakpoints

* adding a test

* Put back resolved hostname

* ficed resolved host name

* Fixed more resolved host name

* Imporved unit test

* trying to add deepdiff pkg

* Fixed test to combine payloads

* added deepdiff to the sqlserver hatch

* Tried to add deepdifff deferently

* Enabled test

* Added a total limit of columns

* Improved exception treatment

* fixed hostname

* Added Foreign key columns

* Added Foreign key columns

* Sorted tables

* add time log for individual query

* removed other jobs

* Added timestamps

* Add more logs

* increase to 500

* removing postgres simulation

* fix errors

* added collection interval

* Added arrays to indexes nd partitions

* added error logs

* formatted queries

* format queries

* refactored queries execution

* improved formatting

* applied lnter

* Updated test expectations

* Removed pdb

* Adding a changelog

* removed pdb

* Formatted

* removed populate

* Clean up empty lines

* put back the driver

* put remove check

* put back space

* remove space

* reapplied linter

* Improved changelog

* Improved docs

* improved example

* corrected comment

* added submitter unit test

* xchanged config

* Added param query

* improved logging

* changelog changed

* Added stop iteration

* pujt back odb

* Inherited from async job

* Added conf parameters

* Fixed unit test

* removed pdb

* Formatted comments

* Added a chnage to dbmasync

* Update spec

* Removed pdb

* put back driver

* fixed changelogs

* applied linter

* minor improvments

* fixed typo

* removed base change

* Moved do for db in schemas

* Improved const

* Applied linter

* Improved specs

* added more tests

* Improved doc

* improve variable names

* Applied linter

* linter

* Added test for truncation

* Add db to the message

* Fixed unit test

* Applied linter

* Changed truncation msg

* applied linter

* Require base package version

* Removed deepdiff from ddev hatch

* resolved errors after merge

* remove modification from base

* removed white space

* removed white space again

* synced example

* Added a license

* Put correct date in license

* applied model sync

* create a dedicated test db for schemas

* applied linter

* added test schema db to all envs

* lint test

* normalized ids

* convert to bool windows value

* fix convert function

* fixed put back index row

* Make test agnostic to order of index columns

* updated with latest ddev

Author: kozlovb

sqlserver/assets/configuration/spec.yaml

@@ -713,6 +713,30 @@ files:
         type: number
         example: 1800
         display_default: false
+    - name: schemas_collection 
+      description: |
+        Configure collection of schemas. If `database_autodiscovery` is not enabled, data is collected 
+        only for the database configured with `database` parameter.
+      options:
+        - name: enabled
+          description: |
+            Enable schema collection. Requires `dbm: true`. Defaults to false.
+          value:
+            type: boolean
+            example: false
+        - name: collection_interval
+          description: |
+            Set the database schema collection interval (in seconds). Defaults to 600 seconds.
+          value:
+            type: number
+            example: 600
+        - name: max_execution_time
+          description: |
+            Set the maximum time for schema collection (in seconds). Defaults to 10 seconds. 
+            Capped by `schemas_collection.collection_interval`
+          value:
+            type: number
+            example: 10
     - template: instances/default
   - template: logs
     example:

sqlserver/changelog.d/17258.added

@@ -0,0 +1,2 @@
+Adding schema collection to sqlserver
+Schema data includes information about the tables, their columns, indexes, foreign keys, and partitions.

sqlserver/datadog_checks/sqlserver/config.py

@@ -7,7 +7,10 @@
 
 from datadog_checks.base.config import is_affirmative
 from datadog_checks.base.utils.common import to_native_string
-from datadog_checks.sqlserver.const import DEFAULT_AUTODISCOVERY_INTERVAL, PROC_CHAR_LIMIT
+from datadog_checks.sqlserver.const import (
+    DEFAULT_AUTODISCOVERY_INTERVAL,
+    PROC_CHAR_LIMIT,
+)
 
 
 class SQLServerConfig:
@@ -45,6 +48,7 @@ def __init__(self, init_config, instance, log):
         self.procedure_metrics_config: dict = instance.get('procedure_metrics', {}) or {}
         self.settings_config: dict = instance.get('collect_settings', {}) or {}
         self.activity_config: dict = instance.get('query_activity', {}) or {}
+        self.schema_config: dict = instance.get('schemas_collection', {}) or {}
         self.cloud_metadata: dict = {}
         aws: dict = instance.get('aws', {}) or {}
         gcp: dict = instance.get('gcp', {}) or {}

sqlserver/datadog_checks/sqlserver/config_models/instance.py

@@ -139,6 +139,16 @@ class QueryMetrics(BaseModel):
     samples_per_hour_per_query: Optional[int] = None
 
 
+class SchemasCollection(BaseModel):
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        frozen=True,
+    )
+    collection_interval: Optional[float] = None
+    enabled: Optional[bool] = None
+    max_execution_time: Optional[float] = None
+
+
 class InstanceConfig(BaseModel):
     model_config = ConfigDict(
         validate_default=True,
@@ -199,6 +209,7 @@ class InstanceConfig(BaseModel):
     query_activity: Optional[QueryActivity] = None
     query_metrics: Optional[QueryMetrics] = None
     reported_hostname: Optional[str] = None
+    schemas_collection: Optional[SchemasCollection] = None
     server_version: Optional[str] = None
     service: Optional[str] = None
     stored_procedure: Optional[str] = None

sqlserver/datadog_checks/sqlserver/const.py

@@ -268,3 +268,5 @@
 ]
 
 PROC_CHAR_LIMIT = 500
+
+DEFAULT_SCHEMAS_COLLECTION_INTERVAL = 600

sqlserver/datadog_checks/sqlserver/data/conf.yaml.example

@@ -659,6 +659,27 @@ instances:
     #
     # ignore_missing_database: false
 
+    ## Configure collection of schemas. If `database_autodiscovery` is not enabled, data is collected 
+    ## only for the database configured with `database` parameter.
+    #
+    # schemas_collection:
+
+        ## @param enabled - boolean - optional - default: false
+        ## Enable schema collection. Requires `dbm: true`. Defaults to false.
+        #
+        # enabled: false
+
+        ## @param collection_interval - number - optional - default: 600
+        ## Set the database schema collection interval (in seconds). Defaults to 600 seconds.
+        #
+        # collection_interval: 600
+
+        ## @param max_execution_time - number - optional - default: 10
+        ## Set the maximum time for schema collection (in seconds). Defaults to 10 seconds. 
+        ## Capped by `schemas_collection.collection_interval`
+        #
+        # max_execution_time: 10
+
     ## @param tags - list of strings - optional
     ## A list of tags to attach to every metric and service check emitted by this instance.
     ##

sqlserver/datadog_checks/sqlserver/queries.py

@@ -143,6 +143,73 @@
     ],
 }
 
+DB_QUERY = """
+SELECT
+    db.database_id AS id, db.name AS name, db.collation_name AS collation, dp.name AS owner
+FROM
+    sys.databases db LEFT JOIN sys.database_principals dp ON db.owner_sid = dp.sid
+WHERE db.name IN ({});
+"""
+
+SCHEMA_QUERY = """
+SELECT
+    s.name AS name, s.schema_id AS id, dp.name AS owner_name
+FROM
+    sys.schemas AS s JOIN sys.database_principals dp ON s.principal_id = dp.principal_id
+WHERE s.name NOT IN ('sys', 'information_schema')
+"""
+
+TABLES_IN_SCHEMA_QUERY = """
+SELECT
+    object_id AS id, name
+FROM
+    sys.tables
+WHERE schema_id=?
+"""
+
+COLUMN_QUERY = """
+SELECT
+    column_name AS name, data_type, column_default, is_nullable AS nullable , table_name, ordinal_position
+FROM
+    information_schema.columns
+WHERE
+    table_name IN ({}) and table_schema='{}';
+"""
+
+PARTITIONS_QUERY = """
+SELECT
+    object_id AS id, COUNT(*) AS partition_count
+FROM
+    sys.partitions
+WHERE
+    object_id IN ({}) GROUP BY object_id;
+"""
+
+INDEX_QUERY = """
+SELECT
+    i.object_id AS id, i.name, i.type, i.is_unique, i.is_primary_key, i.is_unique_constraint,
+    i.is_disabled, STRING_AGG(c.name, ',') AS column_names
+FROM
+    sys.indexes i JOIN sys.index_columns ic ON i.object_id = ic.object_id
+    AND i.index_id = ic.index_id JOIN sys.columns c ON ic.object_id = c.object_id AND ic.column_id = c.column_id
+WHERE
+    i.object_id IN ({}) GROUP BY i.object_id, i.name, i.type,
+    i.is_unique, i.is_primary_key, i.is_unique_constraint, i.is_disabled;
+"""
+
+FOREIGN_KEY_QUERY = """
+SELECT
+    FK.referenced_object_id AS id, FK.name AS foreign_key_name,
+    OBJECT_NAME(FK.parent_object_id) AS referencing_table,
+    STRING_AGG(COL_NAME(FKC.parent_object_id, FKC.parent_column_id),',') AS referencing_column,
+    OBJECT_NAME(FK.referenced_object_id) AS referenced_table,
+    STRING_AGG(COL_NAME(FKC.referenced_object_id, FKC.referenced_column_id),',') AS referenced_column
+FROM
+    sys.foreign_keys AS FK JOIN sys.foreign_key_columns AS FKC ON FK.object_id = FKC.constraint_object_id
+WHERE
+    FK.referenced_object_id IN ({}) GROUP BY FK.name, FK.parent_object_id, FK.referenced_object_id;
+"""
+
 
 def get_query_ao_availability_groups(sqlserver_major_version):
     """