DBMON-5416 Collect table size metrics for SQLServer (#20367)

* Sqlserver table size metrics

* wip

* Wip

* Collect table size metrics for SQLServer

* Format

* Exclude tempdb

* Changelog

* Validate

* Integration test

* Style

* Metadata file

* License

* Skip table size metrics for windows

* Lint

Author: edengorevoy

sqlserver/assets/configuration/spec.yaml

@@ -430,6 +430,23 @@ files:
             - name: enabled
               type: boolean
               example: false
+        - name: table_size_metrics
+          description: |
+            Configure collection of table size metrics.
+
+            Set `table_size_metrics.enabled` to true to enable collection of table size metrics. 
+            Does not collect tempdb metrics.
+            Defaults to false.
+          value:
+            type: object
+            properties:
+            - name: enabled
+              type: boolean
+              example: false
+            - name: collection_interval
+              type: integer
+              example: 600
+              display_default: 600
     - name: agent_jobs
       description: Configure collection of SQL Server Agent jobs events and metrics
       options:

sqlserver/changelog.d/20367.added

@@ -0,0 +1 @@
+Add size and row count collection for tables in SQLServer

sqlserver/datadog_checks/sqlserver/config.py

@@ -217,6 +217,7 @@ def _build_database_metrics_configs(self, instance):
             "task_scheduler_metrics": {'enabled': False},
             "tempdb_file_space_usage_metrics": {'enabled': True},
             "xe_metrics": {'enabled': False},
+            "table_size_metrics": {'enabled': False, 'collection_interval': DEFAULT_LONG_METRICS_COLLECTION_INTERVAL},
         }
         # Check if the instance has any configuration for the metrics in legacy structure
         legacy_configuration_metrics = {

sqlserver/datadog_checks/sqlserver/config_models/instance.py

@@ -197,6 +197,15 @@ class ServerStateMetrics(BaseModel):
     enabled: Optional[bool] = Field(None, examples=[True])
 
 
+class TableSizeMetrics(BaseModel):
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        frozen=True,
+    )
+    collection_interval: Optional[int] = Field(None, examples=[600])
+    enabled: Optional[bool] = Field(None, examples=[False])
+
+
 class TaskSchedulerMetrics(BaseModel):
     model_config = ConfigDict(
         arbitrary_types_allowed=True,
@@ -239,6 +248,7 @@ class DatabaseMetrics(BaseModel):
     primary_log_shipping_metrics: Optional[PrimaryLogShippingMetrics] = None
     secondary_log_shipping_metrics: Optional[SecondaryLogShippingMetrics] = None
     server_state_metrics: Optional[ServerStateMetrics] = None
+    table_size_metrics: Optional[TableSizeMetrics] = None
     task_scheduler_metrics: Optional[TaskSchedulerMetrics] = None
     tempdb_file_space_usage_metrics: Optional[TempdbFileSpaceUsageMetrics] = None
     xe_metrics: Optional[XeMetrics] = None

sqlserver/datadog_checks/sqlserver/const.py

@@ -253,6 +253,13 @@
     ('sqlserver.database.master_files.state', 'sys.master_files', 'state'),
 ]
 
+TABLE_SIZE_METRICS = [
+    ('sqlserver.table.row_count', 'sys.dm_db_partition_stats', 'row_count'),
+    ('sqlserver.table.total_size', 'sys.dm_db_partition_stats', 'total_size'),
+    ('sqlserver.table.used_size', 'sys.dm_db_partition_stats', 'used_size'),
+    ('sqlserver.table.data_size', 'sys.dm_db_partition_stats', 'data_size'),
+]
+
 TEMPDB_FILE_SPACE_USAGE_METRICS = [
     ('sqlserver.tempdb.file_space_usage.free_space', 'sys.dm_db_file_space_usage', 'free_space'),
     (

sqlserver/datadog_checks/sqlserver/data/conf.yaml.example

@@ -290,6 +290,15 @@ instances:
         #
         # xe_metrics: {}
 
+        ## @param table_size_metrics - mapping - optional
+        ## Configure collection of table size metrics.
+        ##
+        ## Set `table_size_metrics.enabled` to true to enable collection of table size metrics. 
+        ## Does not collect tempdb metrics.
+        ## Defaults to false.
+        #
+        # table_size_metrics: {}
+
     ## Configure collection of SQL Server Agent jobs events and metrics
     #
     # agent_jobs:

sqlserver/datadog_checks/sqlserver/database_metrics/__init__.py

@@ -21,3 +21,4 @@
 from .server_state_metrics import SqlserverServerStateMetrics
 from .tempdb_file_space_usage_metrics import SqlserverTempDBFileSpaceUsageMetrics
 from .xe_session_metrics import SQLServerXESessionMetrics
+from .table_size_metrics import SqlserverTableSizeMetrics

sqlserver/datadog_checks/sqlserver/database_metrics/table_size_metrics.py

@@ -0,0 +1,101 @@
+# (C) Datadog, Inc. 2025-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+import functools
+
+from datadog_checks.base.errors import ConfigurationError
+
+from .base import SqlserverDatabaseMetricsBase
+
+TABLE_SIZE_STATS_QUERY = {
+    "name": "sys.dm_db_partition_stats",
+    "query": """
+    SELECT
+        t.name AS table_name,
+        s.name AS schema_name,
+        db_name() AS database_name,
+        SUM(p.row_count) AS row_count,
+        CAST(SUM(a.total_pages) * 8.0 AS DECIMAL(18,2)) AS total_size,
+        CAST(SUM(a.used_pages) * 8.0 AS DECIMAL(18,2)) AS used_size,
+        CAST(SUM(a.data_pages) * 8.0 AS DECIMAL(18,2)) AS data_size
+    FROM
+        sys.tables t
+    INNER JOIN
+        sys.schemas s ON t.schema_id = s.schema_id
+    INNER JOIN
+        sys.indexes i ON t.object_id = i.object_id
+    INNER JOIN
+        sys.dm_db_partition_stats p ON i.object_id = p.object_id AND i.index_id = p.index_id
+    INNER JOIN
+        sys.allocation_units a ON p.partition_id = a.container_id
+    GROUP BY
+        t.name, s.name
+""",
+    "columns": [
+        {"name": "table", "type": "tag"},
+        {"name": "schema", "type": "tag"},
+        {"name": "database", "type": "tag"},
+        {"name": "table.row_count", "type": "gauge"},
+        {"name": "table.total_size", "type": "gauge"},
+        {"name": "table.used_size", "type": "gauge"},
+        {"name": "table.data_size", "type": "gauge"},
+    ],
+}
+
+
+# https://learn.microsoft.com/en-us/sql/relational-databases/system-dynamic-management-views/sys-dm-db-index-usage-stats-transact-sql?view=sql-server-ver15
+class SqlserverTableSizeMetrics(SqlserverDatabaseMetricsBase):
+    @property
+    def include_table_size_metrics(self) -> bool:
+        return self.config.database_metrics_config["table_size_metrics"]["enabled"]
+
+    @property
+    def collection_interval(self) -> int:
+        '''
+        Returns the interval in seconds at which to collect table size metrics.
+        Note: The table size metrics query can be expensive, so it is recommended to set a higher interval.
+        '''
+        return self.config.database_metrics_config["table_size_metrics"]["collection_interval"]
+
+    @property
+    def databases(self):
+        '''
+        Returns a list of databases to collect table size metrics for.
+        tempdb is excluded.
+        '''
+        if not self._databases:
+            raise ConfigurationError("No databases configured for table size metrics")
+        if 'tempdb' in self._databases:
+            self._databases.remove('tempdb')
+        return self._databases
+
+    @property
+    def enabled(self):
+        if not self.include_table_size_metrics:
+            return False
+        return True
+
+    @property
+    def queries(self):
+        # make a copy of the query to avoid modifying the original
+        # in case different instances have different collection intervals
+        query = TABLE_SIZE_STATS_QUERY.copy()
+        query['collection_interval'] = self.collection_interval
+        return [query]
+
+    def __repr__(self) -> str:
+        return (
+            f"{self.__class__.__name__}(" f"enabled={self.enabled}, " f"collection_interval={self.collection_interval})"
+        )
+
+    def _build_query_executors(self):
+        executors = []
+        for database in self.databases:
+            executor = self.new_query_executor(
+                self.queries,
+                executor=functools.partial(self.execute_query_handler, db=database),
+                track_operation_time=self.track_operation_time,
+            )
+            executor.compile_queries()
+            executors.append(executor)
+        return executors

sqlserver/datadog_checks/sqlserver/sqlserver.py

@@ -44,6 +44,7 @@
     SqlserverPrimaryLogShippingMetrics,
     SqlserverSecondaryLogShippingMetrics,
     SqlserverServerStateMetrics,
+    SqlserverTableSizeMetrics,
     SqlserverTempDBFileSpaceUsageMetrics,
     SQLServerXESessionMetrics,
 )
@@ -887,6 +888,7 @@ def _database_level_database_metrics(self):
             SqlserverIndexUsageMetrics,
             SqlserverDBFragmentationMetrics,
             SqlserverDatabaseFilesMetrics,
+            SqlserverTableSizeMetrics,
         ]
 
     @property

sqlserver/metadata.csv

@@ -141,6 +141,10 @@ sqlserver.stats.procs_blocked,gauge,,process,,The number of processes blocked. (
 sqlserver.stats.safe_auto_param_attempts,gauge,,attempt,,Number of safe auto-parameterization attempts per second. (Perf. Counter: `SQL Statistics - Safe Auto-Params/sec`),0,sql_server,stats safe auto param,,
 sqlserver.stats.sql_compilations,gauge,,operation,second,The number of SQL compilations per second. (Perf. Counter: `SQL Statistics - SQL Compilations/sec`),0,sql_server,sql compils,,
 sqlserver.stats.sql_recompilations,gauge,,operation,second,The number of SQL re-compilations per second. (Perf. Counter: `SQL Statistics - SQL Re-Compilations/sec`),-1,sql_server,sql recompils,,
+sqlserver.table.data_size,gauge,,kibibyte,,The size in kibibytes of data stored in this table excluding internal index pages and allocation-management pages. Tags: `database` `schema` `table`,0,sql_server,total table size,,
+sqlserver.table.row_count,gauge,,row,,The number of rows in this table. Tags: `database` `schema` `table`,0,sql_server,table row count,,
+sqlserver.table.total_size,gauge,,kibibyte,,The total size of the table in kibibytes. Tags: `database` `schema` `table`,0,sql_server,table data size,,
+sqlserver.table.used_size,gauge,,kibibyte,,The size in kibibytes of data stored in this table including internal index pages and allocation-management pages. Tags: `database` `schema` `table`,0,sql_server,table used size,,
 sqlserver.task.context_switches_count,gauge,,unit,,Number of scheduler context switches that this task has completed. Tags: 'scheduler_id',0,sql_server,context switch,,
 sqlserver.task.pending_io_byte_average,gauge,,byte,,Average byte count of I/Os that are performed by this task. Tags: 'scheduler_id',0,sql_server,task avg io byte,,
 sqlserver.task.pending_io_byte_count,gauge,,byte,,Total byte count of I/Os that are performed by this task. Tags: 'scheduler_id',0,sql_server,task io byte,,
@@ -154,5 +158,5 @@ sqlserver.transactions.longest_transaction_running_time,gauge,,second,,The time
 sqlserver.transactions.version_cleanup_rate,gauge,,kibibyte,second,The cleanup rate of the version store in tempdb. (Perf. Counter: `Transactions - Version Cleanup rate (KB/s)`),0,sql_server,version cleanup rate,,
 sqlserver.transactions.version_generation_rate,gauge,,kibibyte,second,The generation rate of the version store in tempdb. (Perf. Counter: `Transactions - Version Generation rate (KB/s)`),0,sql_server,version generation rate,,
 sqlserver.transactions.version_store_size,gauge,,kibibyte,,The size of the version store in tempdb. (Perf. Counter: `Transactions - Version Store Size (KB)`),0,sql_server,version store size,,
-sqlserver.xe.events_not_in_xml,gauge,,event,,"Number of generated events that are missing in the XML representation of the ring buffer. Tags: `session_name`",0,sql_server,xe events not in xml,,
-sqlserver.xe.session_status,gauge,,,,"Status of the node in a SQL Server failover cluster instance. Tags: `session_name`",0,sql_server,xe session status,,
+sqlserver.xe.events_not_in_xml,gauge,,event,,Number of generated events that are missing in the XML representation of the ring buffer. Tags: `session_name`,0,sql_server,xe events not in xml,,
+sqlserver.xe.session_status,gauge,,,,Status of the node in a SQL Server failover cluster instance. Tags: `session_name`,0,sql_server,xe session status,,

sqlserver/tests/common.py

@@ -22,6 +22,7 @@
     INSTANCE_METRICS,
     INSTANCE_METRICS_DATABASE_AO,
     INSTANCE_METRICS_DATABASE_SINGLE,
+    TABLE_SIZE_METRICS,
     TASK_SCHEDULER_METRICS,
     TEMPDB_FILE_SPACE_USAGE_METRICS,
 )
@@ -97,6 +98,7 @@ def get_local_driver():
             DATABASE_METRICS,
             DATABASE_BACKUP_METRICS,
             TEMPDB_FILE_SPACE_USAGE_METRICS,
+            TABLE_SIZE_METRICS,
         )
     ]
     + DATABASE_INDEX_METRICS

sqlserver/tests/conftest.py

@@ -115,6 +115,9 @@ def instance_docker(instance_docker_defaults):
                 'master_files_metrics': {
                     'enabled': True,
                 },
+                'table_size_metrics': {
+                    'enabled': True,
+                },
             },
             'disable_generic_tags': True,
         }

sqlserver/tests/test_database_metrics.py

@@ -32,6 +32,7 @@
     SqlserverPrimaryLogShippingMetrics,
     SqlserverSecondaryLogShippingMetrics,
     SqlserverServerStateMetrics,
+    SqlserverTableSizeMetrics,
     SqlserverTempDBFileSpaceUsageMetrics,
 )
 from datadog_checks.sqlserver.utils import Database
@@ -920,7 +921,6 @@ def test_sqlserver_db_fragmentation_metrics(
         instance_docker_metrics['database_metrics']['db_fragmentation_metrics'][
             'collection_interval'
         ] = db_fragmentation_metrics_interval
-    print(instance_docker_metrics)
     mocked_results = [
         [
             ('master', 'spt_fallback_db', 'dbo', 0, None, 0, 0.0, 0, 0.0),
@@ -1301,6 +1301,63 @@ def test_sqlserver_database_files_metrics(
                     aggregator.assert_metric(metric_name, value=metric_value, tags=expected_tags)
 
 
+@pytest.mark.integration
+@pytest.mark.usefixtures('dd_environment')
+@pytest.mark.parametrize('include_table_size_metrics', [True, False])
+def test_sqlserver_table_size_metrics(
+    aggregator,
+    dd_run_check,
+    init_config,
+    instance_docker_metrics,
+    include_table_size_metrics,
+):
+    instance_docker_metrics['database_autodiscovery'] = True
+    instance_docker_metrics['database_metrics'] = {
+        'table_size_metrics': {'enabled': include_table_size_metrics},
+    }
+
+    mocked_results = [
+        ('table1', 'dbo', 'master', 100, 1024, 500, 200),
+        ('table2', 'dbo', 'master', 100, 1024, 500, 200),
+        ('table3', 'dbo', 'master', 100, 1024, 500, 200),
+        ('table4', 'dbo', 'datadog_test-1', 100, 1024, 500, 200),
+    ]
+
+    sqlserver_check = SQLServer(CHECK_NAME, init_config, [instance_docker_metrics])
+    sqlserver_check.databases = {Database(db) for db in AUTODISCOVERY_DBS}
+
+    execute_query_handler_mocked = mock.MagicMock()
+    execute_query_handler_mocked.return_value = mocked_results
+
+    table_size_metrics = SqlserverTableSizeMetrics(
+        config=sqlserver_check._config,
+        new_query_executor=sqlserver_check._new_query_executor,
+        server_static_info=STATIC_SERVER_INFO,
+        execute_query_handler=execute_query_handler_mocked,
+        databases=AUTODISCOVERY_DBS,
+    )
+
+    sqlserver_check._database_metrics = [table_size_metrics]
+
+    dd_run_check(sqlserver_check)
+
+    if not include_table_size_metrics:
+        assert table_size_metrics.enabled is False
+    else:
+        tags = sqlserver_check._config.tags
+        for mock_row in mocked_results:
+            table_name, schema, database, *metric_values = mock_row
+            metrics = zip(table_size_metrics.metric_names()[0], metric_values)
+            expected_tags = [
+                f'table:{table_name}',
+                f'schema:{schema}',
+                f'database:{database}',
+            ] + tags
+            # check that the aggregator got the mocked metrics
+            for metric_name, metric_value in metrics:
+                aggregator.assert_metric(metric_name, value=metric_value, tags=expected_tags)
+
+
 @pytest.mark.integration
 @pytest.mark.usefixtures('dd_environment')
 @pytest.mark.parametrize('include_database_files_metrics', [True, False])

sqlserver/tests/test_integration.py

@@ -20,6 +20,7 @@
     STATIC_INFO_ENGINE_EDITION,
     STATIC_INFO_MAJOR_VERSION,
     STATIC_INFO_VERSION,
+    TABLE_SIZE_METRICS,
 )
 
 from .common import (
@@ -452,7 +453,7 @@ def test_check_windows_defaults(aggregator, dd_run_check, init_config, instance_
     for mname in EXPECTED_DEFAULT_METRICS + CUSTOM_METRICS:
 
         # These require extra setup to test
-        if mname not in DATABASE_INDEX_METRICS:
+        if mname not in DATABASE_INDEX_METRICS and mname not in [m[0] for m in TABLE_SIZE_METRICS]:
             aggregator.assert_metric(mname)
 
     aggregator.assert_service_check('sqlserver.can_connect', status=SQLServer.OK)
@@ -486,6 +487,43 @@ def test_index_fragmentation_metrics(aggregator, dd_run_check, instance_docker,
         assert 'datadog_test-1' in seen_databases
 
 
+@pytest.mark.integration
+@pytest.mark.usefixtures('dd_environment')
+@pytest.mark.parametrize('database_autodiscovery', [True, False])
+def test_table_size_metrics(aggregator, dd_run_check, instance_docker, database_autodiscovery):
+    instance_docker['database_autodiscovery'] = database_autodiscovery
+    sqlserver_check = SQLServer(CHECK_NAME, {}, [instance_docker])
+    dd_run_check(sqlserver_check)
+    seen_databases = set()
+    for m in aggregator.metrics("sqlserver.table.row_count"):
+        tags_by_key = dict([t.split(':') for t in m.tags if not t.startswith('dd.internal')])
+        seen_databases.add(tags_by_key['database'])
+        assert tags_by_key['table'].lower() != 'none'
+        assert tags_by_key['schema'].lower() != 'none'
+
+    for m in aggregator.metrics("sqlserver.table.data_size"):
+        tags_by_key = dict([t.split(':') for t in m.tags if not t.startswith('dd.internal')])
+        seen_databases.add(tags_by_key['database'])
+        assert tags_by_key['table'].lower() != 'none'
+        assert tags_by_key['schema'].lower() != 'none'
+
+    for m in aggregator.metrics("sqlserver.table.total_size"):
+        tags_by_key = dict([t.split(':') for t in m.tags if not t.startswith('dd.internal')])
+        seen_databases.add(tags_by_key['database'])
+        assert tags_by_key['table'].lower() != 'none'
+        assert tags_by_key['schema'].lower() != 'none'
+
+    for m in aggregator.metrics("sqlserver.table.used_size"):
+        tags_by_key = dict([t.split(':') for t in m.tags if not t.startswith('dd.internal')])
+        seen_databases.add(tags_by_key['database'])
+        assert tags_by_key['table'].lower() != 'none'
+        assert tags_by_key['schema'].lower() != 'none'
+
+    assert 'master' in seen_databases
+    if database_autodiscovery:
+        assert 'datadog_test-1' in seen_databases
+
+
 @pytest.mark.integration
 @pytest.mark.usefixtures('dd_environment')
 def test_custom_metrics_fraction_counters(aggregator, dd_run_check, instance_docker, caplog):