Skip to content

Commit db9b87e

Browse files
authored
Revert "[SDBM-842] Postgres integration performance optimization: Limit how many records are pulled from pg_stat_statements. (#17187)" (#17397)
This reverts commit 798f2f4.
1 parent c998c94 commit db9b87e

File tree

4 files changed

+6
-142
lines changed

4 files changed

+6
-142
lines changed

postgres/changelog.d/17397.fixed

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Revert Postgres Optimization (#17187).
2+
3+
This appears to lead to inflated metrics in certain cases. Removing this optimization while we fix the inflated metrics.

postgres/datadog_checks/postgres/query_calls_cache.py

Lines changed: 0 additions & 37 deletions
This file was deleted.

postgres/datadog_checks/postgres/statements.py

Lines changed: 3 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@
1919
from datadog_checks.base.utils.tracking import tracked_method
2020
from datadog_checks.postgres.cursor import CommenterCursor, CommenterDictCursor
2121

22-
from .query_calls_cache import QueryCallsCache
2322
from .util import DatabaseConfigurationError, payload_pg_version, warning_with_tags
2423
from .version_utils import V9_4, V14
2524

@@ -28,21 +27,15 @@
2827
except ImportError:
2928
from ..stubs import datadog_agent
3029

31-
QUERYID_TO_CALLS_QUERY = """
32-
SELECT queryid, calls
33-
FROM {pg_stat_statements_view}
34-
WHERE queryid IS NOT NULL
35-
"""
36-
3730
STATEMENTS_QUERY = """
3831
SELECT {cols}
3932
FROM {pg_stat_statements_view} as pg_stat_statements
4033
LEFT JOIN pg_roles
4134
ON pg_stat_statements.userid = pg_roles.oid
4235
LEFT JOIN pg_database
4336
ON pg_stat_statements.dbid = pg_database.oid
44-
WHERE query NOT LIKE 'EXPLAIN %%'
45-
AND queryid = ANY('{{ {called_queryids} }}'::bigint[])
37+
WHERE query != '<insufficient privilege>'
38+
AND query NOT LIKE 'EXPLAIN %%'
4639
{filters}
4740
{extra_clauses}
4841
"""
@@ -153,7 +146,6 @@ def __init__(self, check, config, shutdown_callback):
153146
self.tags = None
154147
self._state = StatementMetrics()
155148
self._stat_column_cache = []
156-
self._query_calls_cache = QueryCallsCache()
157149
self._track_io_timing_cache = None
158150
self._obfuscate_options = to_native_string(json.dumps(self._config.obfuscator_options))
159151
# full_statement_text_cache: limit the ingestion rate of full statement text events per query_signature
@@ -185,11 +177,7 @@ def _get_pg_stat_statements_columns(self):
185177

186178
# Querying over '*' with limit 0 allows fetching only the column names from the cursor without data
187179
query = STATEMENTS_QUERY.format(
188-
cols='*',
189-
pg_stat_statements_view=self._config.pg_stat_statements_view,
190-
extra_clauses="LIMIT 0",
191-
filters="",
192-
called_queryids="",
180+
cols='*', pg_stat_statements_view=self._config.pg_stat_statements_view, extra_clauses="LIMIT 0", filters=""
193181
)
194182
with self._check._get_main_db() as conn:
195183
with conn.cursor(cursor_factory=CommenterCursor) as cursor:
@@ -198,39 +186,6 @@ def _get_pg_stat_statements_columns(self):
198186
self._stat_column_cache = col_names
199187
return col_names
200188

201-
def _check_called_queries(self):
202-
pgss_view_without_query_text = self._config.pg_stat_statements_view
203-
if pgss_view_without_query_text == "pg_stat_statements":
204-
# Passing false for the showtext argument leads to a huge performance increase. This
205-
# allows the engine to avoid retrieving the potentially large amount of text data.
206-
# The query count query does not depend on the statement text, so it's safe for this use case.
207-
# For more info: https://www.postgresql.org/docs/current/pgstatstatements.html#PGSTATSTATEMENTS-FUNCS
208-
pgss_view_without_query_text = "pg_stat_statements(false)"
209-
210-
with self._check._get_main_db() as conn:
211-
with conn.cursor(cursor_factory=CommenterCursor) as cursor:
212-
called_queryids = []
213-
query = QUERYID_TO_CALLS_QUERY.format(pg_stat_statements_view=pgss_view_without_query_text)
214-
rows = self._execute_query(cursor, query, params=(self._config.dbname,))
215-
for row in rows:
216-
queryid = row[0]
217-
if queryid is None:
218-
continue
219-
220-
calls = row[1]
221-
calls_changed = self._query_calls_cache.set_calls(queryid, calls)
222-
if calls_changed:
223-
called_queryids.append(queryid)
224-
self._query_calls_cache.end_query_call_snapshot()
225-
self._check.gauge(
226-
"dd.postgresql.pg_stat_statements.calls_changed",
227-
len(called_queryids),
228-
tags=self.tags,
229-
hostname=self._check.resolved_hostname,
230-
)
231-
232-
return called_queryids
233-
234189
def run_job(self):
235190
# do not emit any dd.internal metrics for DBM specific check code
236191
self.tags = [t for t in self._tags if not t.startswith('dd.internal')]
@@ -267,7 +222,6 @@ def collect_per_statement_metrics(self):
267222
@tracked_method(agent_check_getter=agent_check_getter, track_result_length=True)
268223
def _load_pg_stat_statements(self):
269224
try:
270-
called_queryids = self._check_called_queries()
271225
available_columns = set(self._get_pg_stat_statements_columns())
272226
missing_columns = PG_STAT_STATEMENTS_REQUIRED_COLUMNS - available_columns
273227
if len(missing_columns) > 0:
@@ -342,7 +296,6 @@ def _load_pg_stat_statements(self):
342296
pg_stat_statements_view=self._config.pg_stat_statements_view,
343297
filters=filters,
344298
extra_clauses="",
345-
called_queryids=', '.join([str(i) for i in called_queryids]),
346299
),
347300
params=params,
348301
)

postgres/tests/test_query_calls_cache.py

Lines changed: 0 additions & 55 deletions
This file was deleted.

0 commit comments

Comments
 (0)