Skip to content

Commit b54df51

Browse files
natashadadanatashadada
authored
PG17 support (#19625)
* PG17 support * changelog * changelog
1 parent 67baf60 commit b54df51

File tree

12 files changed

+158
-20
lines changed

12 files changed

+158
-20
lines changed

‎postgres/changelog.d/19625.added

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Add support for Postgres 17

‎postgres/datadog_checks/postgres/metrics_cache.py

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,20 +15,21 @@
1515
ACTIVITY_QUERY_LT_10,
1616
CHECKSUM_METRICS,
1717
COMMON_ARCHIVER_METRICS,
18-
COMMON_BGW_METRICS,
18+
COMMON_BGW_METRICS_LT_17,
1919
COMMON_METRICS,
2020
DATABASE_SIZE_METRICS,
2121
DBM_MIGRATED_METRICS,
2222
NEWER_14_METRICS,
23-
NEWER_91_BGW_METRICS,
24-
NEWER_92_BGW_METRICS,
23+
NEWER_91_BGW_METRICS_LT_17,
24+
NEWER_92_BGW_METRICS_LT_17,
2525
NEWER_92_METRICS,
26+
QUERY_PG_BGWRITER_CHECKPOINTER,
2627
REPLICATION_METRICS_9_1,
2728
REPLICATION_METRICS_9_2,
2829
REPLICATION_METRICS_10,
2930
REPLICATION_STATS_METRICS,
3031
)
31-
from .version_utils import V8_3, V9, V9_1, V9_2, V9_4, V9_6, V10, V12, V14
32+
from .version_utils import V8_3, V9, V9_1, V9_2, V9_4, V9_6, V10, V12, V14, V17
3233

3334
logger = logging.getLogger(__name__)
3435

@@ -117,14 +118,17 @@ def get_bgw_metrics(self, version):
117118
depending on the postgres version.
118119
Uses a dictionary to save the result for each instance
119120
"""
121+
if version >= V17:
122+
return QUERY_PG_BGWRITER_CHECKPOINTER
123+
120124
# Extended 9.2+ metrics if needed
121125
if self.bgw_metrics is None:
122-
self.bgw_metrics = dict(COMMON_BGW_METRICS)
126+
self.bgw_metrics = dict(COMMON_BGW_METRICS_LT_17)
123127

124128
if version >= V9_1:
125-
self.bgw_metrics.update(NEWER_91_BGW_METRICS)
129+
self.bgw_metrics.update(NEWER_91_BGW_METRICS_LT_17)
126130
if version >= V9_2:
127-
self.bgw_metrics.update(NEWER_92_BGW_METRICS)
131+
self.bgw_metrics.update(NEWER_92_BGW_METRICS_LT_17)
128132

129133
if not self.bgw_metrics:
130134
return None

‎postgres/datadog_checks/postgres/postgres.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
STAT_WAL_METRICS,
6565
SUBSCRIPTION_STATE_METRICS,
6666
VACUUM_PROGRESS_METRICS,
67+
VACUUM_PROGRESS_METRICS_LT_17,
6768
WAL_FILE_METRICS,
6869
DatabaseConfigurationError,
6970
DatabaseHealthCheckError, # noqa: F401
@@ -72,7 +73,7 @@
7273
payload_pg_version,
7374
warning_with_tags,
7475
)
75-
from .version_utils import V9, V9_2, V10, V12, V13, V14, V15, V16, VersionUtils
76+
from .version_utils import V9, V9_2, V10, V12, V13, V14, V15, V16, V17, VersionUtils
7677

7778
try:
7879
import datadog_agent
@@ -314,7 +315,7 @@ def dynamic_queries(self):
314315
if self._config.collect_buffercache_metrics:
315316
queries.append(BUFFERCACHE_METRICS)
316317
queries.append(QUERY_PG_REPLICATION_SLOTS)
317-
queries.append(VACUUM_PROGRESS_METRICS)
318+
queries.append(VACUUM_PROGRESS_METRICS if self.version >= V17 else VACUUM_PROGRESS_METRICS_LT_17)
318319
queries.append(STAT_SUBSCRIPTION_METRICS)
319320

320321
if self.version >= V12:

‎postgres/datadog_checks/postgres/statements.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,12 @@ def statements_query(**kwargs):
8080
PG_STAT_STATEMENTS_REQUIRED_COLUMNS = frozenset({'calls', 'query', 'rows'})
8181

8282
PG_STAT_STATEMENTS_TIMING_COLUMNS = frozenset(
83+
{
84+
'shared_blk_read_time',
85+
'shared_blk_write_time',
86+
}
87+
)
88+
PG_STAT_STATEMENTS_TIMING_COLUMNS_LT_17 = frozenset(
8389
{
8490
'blk_read_time',
8591
'blk_write_time',
@@ -114,6 +120,7 @@ def statements_query(**kwargs):
114120
}
115121
)
116122
| PG_STAT_STATEMENTS_TIMING_COLUMNS
123+
| PG_STAT_STATEMENTS_TIMING_COLUMNS_LT_17
117124
)
118125

119126
PG_STAT_STATEMENTS_TAG_COLUMNS = frozenset(
@@ -350,6 +357,7 @@ def _load_pg_stat_statements(self):
350357

351358
if self._check.pg_settings.get("track_io_timing") != "on":
352359
desired_columns -= PG_STAT_STATEMENTS_TIMING_COLUMNS
360+
desired_columns -= PG_STAT_STATEMENTS_TIMING_COLUMNS_LT_17
353361

354362
pg_stat_statements_max_setting = self._check.pg_settings.get("pg_stat_statements.max")
355363
pg_stat_statements_max = int(

‎postgres/datadog_checks/postgres/util.py

Lines changed: 54 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -228,7 +228,35 @@ def get_list_chunks(lst, n):
228228
],
229229
}
230230

231-
COMMON_BGW_METRICS = {
231+
QUERY_PG_BGWRITER_CHECKPOINTER = {
232+
'name': 'bgw_metrics',
233+
'query': """
234+
SELECT
235+
cp.num_timed,
236+
cp.num_requested,
237+
cp.buffers_written,
238+
bg.buffers_clean,
239+
bg.maxwritten_clean,
240+
bg.buffers_alloc,
241+
cp.write_time,
242+
cp.sync_time
243+
FROM pg_stat_bgwriter bg, pg_stat_checkpointer cp
244+
""".strip(),
245+
'metrics': {
246+
'checkpoints_timed': ('bgwriter.checkpoints_timed', AgentCheck.monotonic_count),
247+
'checkpoints_req': ('bgwriter.checkpoints_requested', AgentCheck.monotonic_count),
248+
'buffers_checkpoint': ('bgwriter.buffers_checkpoint', AgentCheck.monotonic_count),
249+
'buffers_clean': ('bgwriter.buffers_clean', AgentCheck.monotonic_count),
250+
'maxwritten_clean': ('bgwriter.maxwritten_clean', AgentCheck.monotonic_count),
251+
'buffers_alloc': ('bgwriter.buffers_alloc', AgentCheck.monotonic_count),
252+
'checkpoint_write_time': ('bgwriter.write_time', AgentCheck.monotonic_count),
253+
'checkpoint_sync_time': ('bgwriter.sync_time', AgentCheck.monotonic_count),
254+
},
255+
'descriptors': [],
256+
'relation': False,
257+
}
258+
259+
COMMON_BGW_METRICS_LT_17 = {
232260
'checkpoints_timed': ('bgwriter.checkpoints_timed', AgentCheck.monotonic_count),
233261
'checkpoints_req': ('bgwriter.checkpoints_requested', AgentCheck.monotonic_count),
234262
'buffers_checkpoint': ('bgwriter.buffers_checkpoint', AgentCheck.monotonic_count),
@@ -238,9 +266,9 @@ def get_list_chunks(lst, n):
238266
'buffers_alloc': ('bgwriter.buffers_alloc', AgentCheck.monotonic_count),
239267
}
240268

241-
NEWER_91_BGW_METRICS = {'buffers_backend_fsync': ('bgwriter.buffers_backend_fsync', AgentCheck.monotonic_count)}
269+
NEWER_91_BGW_METRICS_LT_17 = {'buffers_backend_fsync': ('bgwriter.buffers_backend_fsync', AgentCheck.monotonic_count)}
242270

243-
NEWER_92_BGW_METRICS = {
271+
NEWER_92_BGW_METRICS_LT_17 = {
244272
'checkpoint_write_time': ('bgwriter.write_time', AgentCheck.monotonic_count),
245273
'checkpoint_sync_time': ('bgwriter.sync_time', AgentCheck.monotonic_count),
246274
}
@@ -502,10 +530,32 @@ def get_list_chunks(lst, n):
502530
],
503531
}
504532

505-
# Requires PG10+
506533
VACUUM_PROGRESS_METRICS = {
507534
'name': 'vacuum_progress_metrics',
508535
'query': """
536+
SELECT v.datname, c.relname, v.phase,
537+
v.heap_blks_total, v.heap_blks_scanned, v.heap_blks_vacuumed,
538+
v.index_vacuum_count, v.max_dead_tuple_bytes, v.num_dead_item_ids
539+
FROM pg_stat_progress_vacuum as v
540+
JOIN pg_class c on c.oid = v.relid
541+
""",
542+
'columns': [
543+
{'name': 'db', 'type': 'tag'},
544+
{'name': 'table', 'type': 'tag'},
545+
{'name': 'phase', 'type': 'tag'},
546+
{'name': 'vacuum.heap_blks_total', 'type': 'gauge'},
547+
{'name': 'vacuum.heap_blks_scanned', 'type': 'gauge'},
548+
{'name': 'vacuum.heap_blks_vacuumed', 'type': 'gauge'},
549+
{'name': 'vacuum.index_vacuum_count', 'type': 'gauge'},
550+
{'name': 'vacuum.max_dead_tuples', 'type': 'gauge'},
551+
{'name': 'vacuum.num_dead_tuples', 'type': 'gauge'},
552+
],
553+
}
554+
555+
# Requires PG10+
556+
VACUUM_PROGRESS_METRICS_LT_17 = {
557+
'name': 'vacuum_progress_metrics',
558+
'query': """
509559
SELECT v.datname, c.relname, v.phase,
510560
v.heap_blks_total, v.heap_blks_scanned, v.heap_blks_vacuumed,
511561
v.index_vacuum_count, v.max_dead_tuples, v.num_dead_tuples

‎postgres/datadog_checks/postgres/version_utils.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
V14 = VersionInfo.parse("14.0.0")
2222
V15 = VersionInfo.parse("15.0.0")
2323
V16 = VersionInfo.parse("16.0.0")
24+
V17 = VersionInfo.parse("17.0.0")
2425

2526

2627
class VersionUtils(object):

‎postgres/hatch.toml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ mypy-deps = [
1212

1313
[[envs.default.matrix]]
1414
python = ["3.12"]
15-
version = ["9.6", "10.0", "11.0", "12.17", "13.0", "14.0", "15.0", "16.0"]
15+
version = ["9.6", "10.0", "11.0", "12.17", "13.0", "14.0", "15.0", "16.0", "17.0"]
1616

1717
[envs.default.overrides]
1818
matrix.version.env-vars = [
@@ -23,6 +23,7 @@ matrix.version.env-vars = [
2323
{ key = "POSTGRES_VERSION", value = "14", if = ["14.0"] },
2424
{ key = "POSTGRES_VERSION", value = "15", if = ["15.0"] },
2525
{ key = "POSTGRES_VERSION", value = "16", if = ["16.0"] },
26+
{ key = "POSTGRES_VERSION", value = "17", if = ["17.0"] },
2627
]
2728

2829
[envs.latest.env-vars]

‎postgres/tests/common.py

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -93,14 +93,13 @@
9393
'postgresql.bgwriter.buffers_checkpoint',
9494
'postgresql.bgwriter.buffers_clean',
9595
'postgresql.bgwriter.maxwritten_clean',
96-
'postgresql.bgwriter.buffers_backend',
9796
'postgresql.bgwriter.buffers_alloc',
98-
'postgresql.bgwriter.buffers_backend_fsync',
9997
'postgresql.bgwriter.write_time',
10098
'postgresql.bgwriter.sync_time',
10199
]
102100

103101
COMMON_BGW_METRICS_PG_ABOVE_94 = ['postgresql.archiver.archived_count', 'postgresql.archiver.failed_count']
102+
COMMON_BGW_METRICS_PG_BELOW_17 = ['postgresql.bgwriter.buffers_backend', 'postgresql.bgwriter.buffers_backend_fsync']
104103
CONNECTION_METRICS = ['postgresql.max_connections', 'postgresql.percent_usage_connections']
105104
CONNECTION_METRICS_DB = ['postgresql.connections']
106105
COMMON_DBS = ['dogs', 'postgres', 'dogs_nofunc', 'dogs_noschema', DB_NAME]
@@ -361,6 +360,10 @@ def check_bgw_metrics(aggregator, expected_tags, count=1):
361360
for name in COMMON_BGW_METRICS:
362361
aggregator.assert_metric(name, count=count, tags=expected_tags)
363362

363+
if float(POSTGRES_VERSION) < 17:
364+
for name in COMMON_BGW_METRICS_PG_BELOW_17:
365+
aggregator.assert_metric(name, count=count, tags=expected_tags)
366+
364367
if float(POSTGRES_VERSION) >= 9.4:
365368
for name in COMMON_BGW_METRICS_PG_ABOVE_94:
366369
aggregator.assert_metric(name, count=count, tags=expected_tags)
@@ -370,7 +373,27 @@ def check_slru_metrics(aggregator, expected_tags, count=1):
370373
if float(POSTGRES_VERSION) < 13.0:
371374
return
372375

373-
slru_caches = ['Subtrans', 'Serial', 'MultiXactMember', 'Xact', 'other', 'Notify', 'CommitTs', 'MultiXactOffset']
376+
slru_caches = [
377+
'subtransaction',
378+
'serializable',
379+
'multixact_member',
380+
'transaction',
381+
'other',
382+
'notify',
383+
'commit_timestamp',
384+
'multixact_offset',
385+
]
386+
if float(POSTGRES_VERSION) < 17.0:
387+
slru_caches = [
388+
'Subtrans',
389+
'Serial',
390+
'MultiXactMember',
391+
'Xact',
392+
'other',
393+
'Notify',
394+
'CommitTs',
395+
'MultiXactOffset',
396+
]
374397
for metric_name in _iterate_metric_name(SLRU_METRICS):
375398
for slru_cache in slru_caches:
376399
aggregator.assert_metric(

‎postgres/tests/test_pg_integration.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1080,7 +1080,8 @@ def assert_state_clean(check):
10801080

10811081
def assert_state_set(check):
10821082
assert check.metrics_cache.instance_metrics
1083-
assert check.metrics_cache.bgw_metrics
1083+
if float(POSTGRES_VERSION) < 17.0:
1084+
assert check.metrics_cache.bgw_metrics
10841085
if POSTGRES_VERSION != '9.3':
10851086
assert check.metrics_cache.archiver_metrics
10861087
assert check.metrics_cache.replication_metrics

‎postgres/tests/test_progress_stats.py

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
CLUSTER_VACUUM_PROGRESS_METRICS,
1111
INDEX_PROGRESS_METRICS,
1212
VACUUM_PROGRESS_METRICS,
13+
VACUUM_PROGRESS_METRICS_LT_17,
1314
)
1415

1516
from .common import DB_NAME, _get_expected_tags, _iterate_metric_name
@@ -20,6 +21,8 @@
2021
lock_table,
2122
requires_over_12,
2223
requires_over_13,
24+
requires_over_17,
25+
requires_under_17,
2326
run_query_thread,
2427
run_vacuum_thread,
2528
)
@@ -72,7 +75,7 @@ def test_analyze_progress(aggregator, integration_check, pg_instance):
7275
aggregator.assert_metric(metric_name, count=1, tags=expected_tags)
7376

7477

75-
@requires_over_12
78+
@requires_over_17
7679
def test_vacuum_progress(aggregator, integration_check, pg_instance):
7780
check = integration_check(pg_instance)
7881

@@ -102,6 +105,37 @@ def test_vacuum_progress(aggregator, integration_check, pg_instance):
102105
aggregator.assert_metric(metric_name, count=1, tags=expected_tags)
103106

104107

108+
@requires_over_12
109+
@requires_under_17
110+
def test_vacuum_progress_lt_17(aggregator, integration_check, pg_instance):
111+
check = integration_check(pg_instance)
112+
113+
# Start vacuum
114+
thread = run_vacuum_thread(pg_instance, 'VACUUM (DISABLE_PAGE_SKIPPING) test_part1')
115+
116+
# Wait for vacuum to be reported
117+
_wait_for_value(
118+
pg_instance,
119+
lower_threshold=0,
120+
query="SELECT count(*) from pg_stat_progress_vacuum",
121+
)
122+
123+
# Collect metrics
124+
check.check(pg_instance)
125+
126+
# Kill vacuum and cleanup thread
127+
kill_vacuum(pg_instance)
128+
thread.join()
129+
130+
expected_tags = _get_expected_tags(check, pg_instance) + [
131+
'phase:scanning heap',
132+
'table:test_part1',
133+
f'db:{DB_NAME}',
134+
]
135+
for metric_name in _iterate_metric_name(VACUUM_PROGRESS_METRICS_LT_17):
136+
aggregator.assert_metric(metric_name, count=1, tags=expected_tags)
137+
138+
105139
@requires_over_12
106140
def test_index_progress(aggregator, integration_check, pg_instance):
107141
check = integration_check(pg_instance)

‎postgres/tests/test_statements.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
from datadog_checks.postgres.statements import (
2727
PG_STAT_STATEMENTS_METRICS_COLUMNS,
2828
PG_STAT_STATEMENTS_TIMING_COLUMNS,
29+
PG_STAT_STATEMENTS_TIMING_COLUMNS_LT_17,
2930
PostgresStatementMetrics,
3031
)
3132
from datadog_checks.postgres.util import payload_pg_version
@@ -297,7 +298,12 @@ def _should_catch_query(dbname):
297298
available_columns = set(row.keys())
298299
metric_columns = available_columns & PG_STAT_STATEMENTS_METRICS_COLUMNS
299300
if track_io_timing_enabled:
300-
assert (available_columns & PG_STAT_STATEMENTS_TIMING_COLUMNS) == PG_STAT_STATEMENTS_TIMING_COLUMNS
301+
if float(POSTGRES_VERSION) >= 17.0:
302+
assert (available_columns & PG_STAT_STATEMENTS_TIMING_COLUMNS) == PG_STAT_STATEMENTS_TIMING_COLUMNS
303+
else:
304+
assert (
305+
available_columns & PG_STAT_STATEMENTS_TIMING_COLUMNS_LT_17
306+
) == PG_STAT_STATEMENTS_TIMING_COLUMNS_LT_17
301307
else:
302308
assert (available_columns & PG_STAT_STATEMENTS_TIMING_COLUMNS) == set()
303309
for col in metric_columns:

‎postgres/tests/utils.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,14 @@
3737
POSTGRES_VERSION is None or float(POSTGRES_VERSION) < 16,
3838
reason='This test is for over 16 only (make sure POSTGRES_VERSION is set)',
3939
)
40+
requires_under_17 = pytest.mark.skipif(
41+
POSTGRES_VERSION is None or float(POSTGRES_VERSION) >= 17,
42+
reason='This test is for under 17 only (make sure POSTGRES_VERSION is set)',
43+
)
44+
requires_over_17 = pytest.mark.skipif(
45+
POSTGRES_VERSION is None or float(POSTGRES_VERSION) < 17,
46+
reason='This test is for over 17 only (make sure POSTGRES_VERSION is set)',
47+
)
4048

4149

4250
def _get_conn(db_instance, dbname=None, user=None, password=None, application_name='test'):

0 commit comments

Comments
 (0)