Add metadata check on PG metrics and fix mismatching metadata type (#20324)

Some metrics were sent as rate but declared as gauge in metadata.csv,
fix metadata to match how they are submitted.

We add additional assertion in common test to check that metadata
matches the reported metrics.

Author: bonnefoa

postgres/metadata.csv

@@ -11,6 +11,8 @@ postgresql.analyze.ext_stats_total,gauge,,,,"Number of extended statistics. This
 postgresql.analyze.sample_blks_scanned,gauge,,block,,"Number of heap blocks scanned. This metric is tagged with db, table, child_relation, phase.",0,postgres,postgres analyze blks scanned,
 postgresql.analyze.sample_blks_total,gauge,,block,,"Total number of heap blocks that will be sampled. This metric is tagged with db, table, child_relation, phase.",0,postgres,postgres analyze blks total,
 postgresql.analyzed,count,,,,"Enabled with `relations`. The number of times this table has been manually analyzed. This metric is tagged with db, schema, table.",0,postgres,analyze,
+postgresql.archiver.archived_count,count,,,,"Number of WAL files that have been successfully archived.",0,postgres,archived cnt,
+postgresql.archiver.failed_count,count,,,,"Number of failed attempts for archiving WAL files.",-1,postgres,failed archived cnt,
 postgresql.autoanalyzed,count,,,,"Enabled with `relations`. The number of times this table has been analyzed by the autovacuum daemon. This metric is tagged with db, schema, table.",0,postgres,auto analyze,
 postgresql.autovacuumed,count,,,,"Enabled with `relations`. The number of times this table has been vacuumed by the autovacuum daemon. This metric is tagged with db, schema, table.",0,postgres,auto vac,
 postgresql.before_xid_wraparound,gauge,,transaction,,The number of transactions that can occur until a transaction wraparound.  This metric is tagged with db.,0,postgres,tx before xid wraparound,
@@ -26,7 +28,7 @@ postgresql.bgwriter.sync_time,count,,millisecond,,The total amount of checkpoint
 postgresql.bgwriter.write_time,count,,millisecond,,The total amount of checkpoint processing time spent writing files to disk.,0,postgres,bgw wrt time,
 postgresql.blk_read_time,count,,millisecond,,Time spent reading data file blocks by backends in this database if track_io_timing is enabled. This metric is tagged with db.,0,postgres,db blk read,
 postgresql.blk_write_time,count,,millisecond,,Time spent writing data file blocks by backends in this database if track_io_timing is enabled. This metric is tagged with db.,0,postgres,db blk write,
-postgresql.buffer_hit,gauge,,hit,second,"The number of times disk blocks were found in the buffer cache, preventing the need to read from the database. This metric is tagged with db.",1,postgres,buff hit,
+postgresql.buffer_hit,rate,,hit,second,"The number of times disk blocks were found in the buffer cache, preventing the need to read from the database. This metric is tagged with db.",1,postgres,buff hit,
 postgresql.buffercache.dirty_buffers,gauge,,buffer,,"Number of dirty shared buffers. pg_buffercache extension needs to be installed. This metric is tagged by db, schema and relation.",0,postgres,buffercache dirty buffers,
 postgresql.buffercache.pinning_backends,gauge,,,,"Number of backends pinning shared buffers. pg_buffercache extension needs to be installed. This metric is tagged by db, schema and relation.",0,postgres,buffercache pinning backends,
 postgresql.buffercache.unused_buffers,gauge,,buffer,,"Number of unused shared buffers. pg_buffercache extension needs to be installed.",0,postgres,buffercache unused buffers,
@@ -39,7 +41,7 @@ postgresql.cluster_vacuum.heap_blks_total,gauge,,block,,"Total number of heap bl
 postgresql.cluster_vacuum.heap_tuples_scanned,gauge,,,,"Number of heap tuples scanned. This counter only advances when the phase is seq scanning heap, index scanning heap or writing new heap. Only available with PostgreSQL 12 and newer. This metric is tagged with db, table, command, phase, index.",0,postgres,postgres cluster tuple_scanned,
 postgresql.cluster_vacuum.heap_tuples_written,gauge,,,,"Number of heap tuples written. This counter only advances when the phase is seq scanning heap, index scanning heap or writing new heap. Only available with PostgreSQL 12 and newer. This metric is tagged with db, table, command, phase, index.",0,postgres,postgres cluster tuple_written,
 postgresql.cluster_vacuum.index_rebuild_count,gauge,,,,"Number of indexes rebuilt. This counter only advances when the phase is rebuilding index. Only available with PostgreSQL 12 and newer. This metric is tagged with db, table, command, phase, index.",0,postgres,postgres cluster rebuild_count,
-postgresql.commits,gauge,,transaction,second,The number of transactions that have been committed in this database. This metric is tagged with db.,0,postgres,commits,
+postgresql.commits,rate,,transaction,second,The number of transactions that have been committed in this database. This metric is tagged with db.,0,postgres,commits,
 postgresql.conflicts.bufferpin,count,,query,,Number of queries in this database that have been canceled due to pinned buffers. Buffer pin conflicts will occur when the walreceiver process tries to apply a buffer cleanup like HOT chain pruning. This require a complete lock of the buffer and any query pinning the buffer will conflict with the cleaning. This metric is tagged with db.,-1,postgres,cfl bufferpin,
 postgresql.conflicts.deadlock,count,,query,,"Number of queries in this database that have been canceled due to deadlocks. Deadlock conflicts will happen when the walreceiver tries to apply a buffer like HOT chain pruning. If the conflict takes more than deadlock_timeout seconds, a deadlock check will be triggered and conflicting queries will be canceled until the buffer is unpinned. This metric is tagged with db.",-1,postgres,cfl deadlock,
 postgresql.conflicts.lock,count,,query,,Number of queries in this database that have been canceled due to lock timeouts. This will occur when the walreceiver process tries to apply a change requiring an ACCESS EXCLUSIVE lock while a query on the replica is reading the table. The conflicting query will be killed after waiting up to max_standby_streaming_delay seconds. This metric is tagged with db.,-1,postgres,cfl lock,
@@ -62,9 +64,9 @@ postgresql.create_index.tuples_total,gauge,,,,"Total number of tuples to be proc
 postgresql.database_size,gauge,,byte,,The disk space used by this database. This metric is tagged with db.,0,postgres,db size,
 postgresql.db.count,gauge,,item,,The number of available databases.,0,postgres,db cnt,
 postgresql.dead_rows,gauge,,row,,"Enabled with `relations`. The estimated number of dead rows. This metric is tagged with db, schema, table.",0,postgres,dead rows,
-postgresql.deadlocks,gauge,,lock,second,The rate of deadlocks detected in this database. This metric is tagged with db.,-1,postgres,deadlocks,
+postgresql.deadlocks,rate,,lock,second,The rate of deadlocks detected in this database. This metric is tagged with db.,-1,postgres,deadlocks,
 postgresql.deadlocks.count,count,,lock,,The number of deadlocks detected in this database. This metric is tagged with db.,-1,postgres,deadlocks.count,
-postgresql.disk_read,gauge,,block,second,The number of disk blocks read in this database. This metric is tagged with db.,0,postgres,disk read,
+postgresql.disk_read,rate,,block,second,The number of disk blocks read in this database. This metric is tagged with db.,0,postgres,disk read,
 postgresql.function.calls,rate,,,,"Enabled with `collect_function_metrics`. The number of calls made to a function. This metric is tagged with db, schema, function.",0,postgres,postgres_function_calls,
 postgresql.function.self_time,rate,,,,"Enabled with `collect_function_metrics`. Total time spent in this function itself, not including other functions called by it. This metric is tagged with db, schema, function.",0,postgres,postgres_function_self_time,
 postgresql.function.total_time,rate,,,,"Enabled with `collect_function_metrics`. Total time spent in this function and all other functions called by it. This metric is tagged with db, schema, function.",0,postgres,postgres_function_total_time,
@@ -139,13 +141,13 @@ postgresql.replication_slot.stream_txns,count,,transaction,,"Number of in-progre
 postgresql.replication_slot.total_bytes,count,,byte,,"Amount of transaction data decoded for sending transactions to the decoding output plugin while decoding changes from WAL for this slot. Extracted from pg_stat_replication_slots. Only available with PostgreSQL 14 and newer. This metric is tagged with slot_name, slot_type, slot_state.",0,postgres,repslot total_bytes,
 postgresql.replication_slot.total_txns,count,,transaction,,"Number of decoded transactions sent to the decoding output plugin for this slot. Extracted from pg_stat_replication_slots. Only available with PostgreSQL 14 and newer. This metric is tagged with slot_name, slot_type, slot_state.",0,postgres,repslot total_txn,
 postgresql.replication_slot.xmin_age,gauge,,transaction,,"The age of the oldest transaction that this slot needs the database to retain. Only physical replication slot will have a xmin. Orphaned replication slot (no consumer or consumer is not connected) will prevent the xmin horizon from progressing. This metric is tagged with slot_name, slot_type, slot_persistence, slot_state.",-1,postgres,repslot xmin,
-postgresql.rollbacks,gauge,,transaction,second,The number of transactions that have been rolled back in this database. This metric is tagged with db.,-1,postgres,rollbacks,
-postgresql.rows_deleted,gauge,,row,second,Enabled with `relations`. The number of rows deleted by queries in this database. This metric is tagged with db.,0,postgres,rows del,
-postgresql.rows_fetched,gauge,,row,second,The number of rows fetched by queries in this database. This metric is tagged with db.,0,postgres,rows fetch,
-postgresql.rows_hot_updated,gauge,,row,second,"Enabled with `relations`. The number of rows HOT updated, meaning no separate index update was needed. This metric is tagged with db, schema, table.",0,postgres,rows hot updated,
-postgresql.rows_inserted,gauge,,row,second,Enabled with `relations`. The number of rows inserted by queries in this database. This metric is tagged with db.,0,postgres,rows insrt,
-postgresql.rows_returned,gauge,,row,second,The number of rows returned by queries in this database. This metric is tagged with db.,0,postgres,rows ret,
-postgresql.rows_updated,gauge,,row,second,Enabled with `relations`. The number of rows updated by queries in this database. This metric is tagged with db.,0,postgres,rows updt,
+postgresql.rollbacks,rate,,transaction,second,The number of transactions that have been rolled back in this database. This metric is tagged with db.,-1,postgres,rollbacks,
+postgresql.rows_deleted,rate,,row,second,Enabled with `relations`. The number of rows deleted by queries in this database. This metric is tagged with db.,0,postgres,rows del,
+postgresql.rows_fetched,rate,,row,second,The number of rows fetched by queries in this database. This metric is tagged with db.,0,postgres,rows fetch,
+postgresql.rows_hot_updated,rate,,row,second,"Enabled with `relations`. The number of rows HOT updated, meaning no separate index update was needed. This metric is tagged with db, schema, table.",0,postgres,rows hot updated,
+postgresql.rows_inserted,rate,,row,second,Enabled with `relations`. The number of rows inserted by queries in this database. This metric is tagged with db.,0,postgres,rows insrt,
+postgresql.rows_returned,rate,,row,second,The number of rows returned by queries in this database. This metric is tagged with db.,0,postgres,rows ret,
+postgresql.rows_updated,rate,,row,second,Enabled with `relations`. The number of rows updated by queries in this database. This metric is tagged with db.,0,postgres,rows updt,
 postgresql.running,gauge,,,,The number of instances running.,0,postgres,running,
 postgresql.seq_rows_read,gauge,,row,second,"Enabled with `relations`. The number of live rows fetched by sequential scans. This metric is tagged with db, schema, table.",0,postgres,seq rows rd,
 postgresql.seq_scans,gauge,,scan,second,"Enabled with `relations`. The number of sequential scans initiated on this table. This metric is tagged with db, schema, table.",0,postgres,seq scans,
@@ -175,8 +177,8 @@ postgresql.subscription.sync_error,count,,,,Number of errors that occurred durin
 postgresql.table.count,gauge,,table,,"The number of user tables in this database. This metric is tagged with db, schema.",0,postgres,tbl count,
 postgresql.table_bloat,gauge,,percent,,"Enabled with `collect_bloat_metrics`. The estimated percentage of table bloat. This metric is tagged with db, schema, table.",0,postgres,tbloat,
 postgresql.table_size,gauge,,byte,,"Enabled with `relations`. The disk space used by the specified table with TOAST data. Free space map and visibility map are not included. This metric is tagged with db, schema, table.",0,postgres,tbl size,
-postgresql.temp_bytes,gauge,,byte,second,The amount of data written to temporary files by queries in this database. This metric is tagged with db.,0,postgres,temp bytes,
-postgresql.temp_files,gauge,,file,second,The number of temporary files created by queries in this database. This metric is tagged with db.,0,postgres,temp files,
+postgresql.temp_bytes,rate,,byte,second,The amount of data written to temporary files by queries in this database. This metric is tagged with db.,0,postgres,temp bytes,
+postgresql.temp_files,rate,,file,second,The number of temporary files created by queries in this database. This metric is tagged with db.,0,postgres,temp files,
 postgresql.toast.autovacuumed,count,,,,"Enabled with `relations`. The number of times the toast table of a relation has been autovacuumed. This metric is tagged with db, schema, table.",0,postgres,toast autovacuumed,
 postgresql.toast.dead_rows,gauge,,,,"Enabled with `relations`. The number of dead rows on the toast table of a relation. This metric is tagged with db, schema, table.",0,postgres,toast dead rows,
 postgresql.toast.index_scans,count,,,,"Enabled with `relations`. The number of index scans done on the toast table of a relation. This metric is tagged with db, schema, table.",0,postgres,toast idx scans,

postgres/tests/common.py

@@ -9,6 +9,7 @@
 from datadog_checks.base.stubs.aggregator import normalize_tags
 from datadog_checks.dev import get_docker_hostname
 from datadog_checks.dev.docker import get_container_ip
+from datadog_checks.dev.utils import get_metadata_metrics
 from datadog_checks.postgres.util import (
     CHECKSUM_METRICS,
     NEWER_14_METRICS,
@@ -500,3 +501,8 @@ def check_stat_io_metrics(aggregator, expected_tags, count=1):
     ]
     for metric_name in _iterate_metric_name(STAT_IO_METRICS):
         aggregator.assert_metric(metric_name, count=count, tags=expected_stat_io_tags)
+
+
+def check_metrics_metadata(aggregator):
+    exclude = ['dd.postgres.operation.time']
+    aggregator.assert_metrics_using_metadata(get_metadata_metrics(), exclude=exclude)

postgres/tests/test_pg_integration.py

@@ -34,6 +34,7 @@
     check_db_count,
     check_file_wal_metrics,
     check_logical_replication_slots,
+    check_metrics_metadata,
     check_performance_metrics,
     check_physical_replication_slots,
     check_slru_metrics,
@@ -97,6 +98,7 @@ def test_common_metrics(aggregator, integration_check, pg_instance, is_aurora):
     check_performance_metrics(aggregator, expected_tags=check.debug_stats_kwargs()['tags'], is_aurora=is_aurora)
 
     aggregator.assert_all_metrics_covered()
+    check_metrics_metadata(aggregator)
 
 
 def _increase_txid(cur):

postgres/tests/test_pg_replication.py

@@ -17,6 +17,7 @@
     check_control_metrics,
     check_db_count,
     check_file_wal_metrics,
+    check_metrics_metadata,
     check_performance_metrics,
     check_replication_delay,
     check_slru_metrics,
@@ -56,6 +57,7 @@ def test_common_replica_metrics(aggregator, integration_check, metrics_cache_rep
     check_performance_metrics(aggregator, expected_tags=check.debug_stats_kwargs()['tags'])
 
     aggregator.assert_all_metrics_covered()
+    check_metrics_metadata(aggregator)
 
 
 @requires_over_10