Skip to content

Commit bfa72f9

Browse files
authored
[mongo] deprecate collStats command and use $collStats aggregation stage to collect collection metrics (#17961)
* use collStats aggregation pipeline to collect collection metrics * add changelog * sort metadata * remove commented code * update test results with new metrics
1 parent 0ee23b6 commit bfa72f9

17 files changed

+2760
-2856
lines changed

mongo/changelog.d/17961.added

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
Added new collection latency and query execution stats metrics.
2+
- mongodb.collection.totalindexsize
3+
- mongodb.collection.collectionscans.nontailable
4+
- mongodb.collection.collectionscans.total
5+
- mongodb.collection.commands.latency
6+
- mongodb.collection.commands.opsps
7+
- mongodb.collection.reads.latency
8+
- mongodb.collection.reads.opsps
9+
- mongodb.collection.transactions.latency
10+
- mongodb.collection.transactions.opsps
11+
- mongodb.collection.writes.latency
12+
- mongodb.collection.writes.opsps

mongo/changelog.d/17961.fixed

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Replace deprecated collStats command with $collStats aggregation stage to collect collection metrics

mongo/datadog_checks/mongo/api.py

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,23 @@ def current_op(self, session=None):
9696
# The $currentOp stage returns a cursor over a stream of documents, each of which reports a single operation.
9797
return self["admin"].aggregate([{'$currentOp': {'allUsers': True}}], session=session)
9898

99+
def coll_stats(self, db_name, coll_name, session=None):
100+
return self[db_name][coll_name].aggregate(
101+
[
102+
{
103+
"$collStats": {
104+
"latencyStats": {},
105+
"storageStats": {},
106+
"queryExecStats": {},
107+
}
108+
},
109+
],
110+
session=session,
111+
)
112+
113+
def index_stats(self, db_name, coll_name, session=None):
114+
return self[db_name][coll_name].aggregate([{"$indexStats": {}}], session=session)
115+
99116
def _is_arbiter(self, options):
100117
cli = MongoClient(**options)
101118
is_master_payload = cli['admin'].command('isMaster')

mongo/datadog_checks/mongo/collectors/coll_stats.py

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -29,20 +29,25 @@ def _get_collections(self, api):
2929
return api.list_authorized_collections(self.db_name)
3030

3131
def collect(self, api):
32-
# Ensure that you're on the right db
33-
db = api[self.db_name]
3432
coll_names = self._get_collections(api)
35-
3633
for coll_name in coll_names:
3734
# Grab the stats from the collection
38-
payload = {'collection': db.command("collstats", coll_name)}
39-
additional_tags = ["db:%s" % self.db_name, "collection:%s" % coll_name]
40-
self._submit_payload(payload, additional_tags, COLLECTION_METRICS)
41-
42-
# Submit the indexSizes metrics manually
43-
index_sizes = payload['collection'].get('indexSizes', {})
44-
metric_name_alias = self._normalize("collection.indexSizes", AgentCheck.gauge)
45-
for idx, val in iteritems(index_sizes):
46-
# we tag the index
47-
idx_tags = self.base_tags + additional_tags + ["index:%s" % idx]
48-
self.gauge(metric_name_alias, val, tags=idx_tags)
35+
for coll_stats in api.coll_stats(self.db_name, coll_name):
36+
# Submit the metrics
37+
storage_stats = coll_stats.get('storageStats', {})
38+
latency_stats = coll_stats.get('latencyStats', {})
39+
query_stats = coll_stats.get('queryExecStats', {})
40+
payload = {'collection': {**storage_stats, **latency_stats, **query_stats}}
41+
additional_tags = ["db:%s" % self.db_name, "collection:%s" % coll_name]
42+
if coll_stats.get('shard'):
43+
# If the collection is sharded, add the shard tag
44+
additional_tags.append("shard:%s" % coll_stats['shard'])
45+
self._submit_payload(payload, additional_tags, COLLECTION_METRICS)
46+
47+
# Submit the indexSizes metrics manually
48+
index_sizes = storage_stats.get('indexSizes', {})
49+
metric_name_alias = self._normalize("collection.indexSizes", AgentCheck.gauge)
50+
for idx, val in iteritems(index_sizes):
51+
# we tag the index
52+
idx_tags = self.base_tags + additional_tags + ["index:%s" % idx]
53+
self.gauge(metric_name_alias, val, tags=idx_tags)

mongo/datadog_checks/mongo/collectors/index_stats.py

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,10 @@ def _get_collections(self, api):
2323
return api.list_authorized_collections(self.db_name)
2424

2525
def collect(self, api):
26-
db = api[self.db_name]
2726
coll_names = self._get_collections(api)
28-
2927
for coll_name in coll_names:
3028
try:
31-
for stats in db[coll_name].aggregate([{"$indexStats": {}}], cursor={}):
29+
for stats in api.index_stats(self.db_name, coll_name):
3230
idx_tags = self.base_tags + [
3331
"name:{0}".format(stats.get('name', 'unknown')),
3432
"collection:{0}".format(coll_name),

mongo/datadog_checks/mongo/metrics.py

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -313,6 +313,7 @@
313313
}
314314

315315
COLLECTION_METRICS = {
316+
# collection storage stats
316317
'collection.size': GAUGE,
317318
'collection.avgObjSize': GAUGE,
318319
'collection.count': GAUGE,
@@ -321,6 +322,19 @@
321322
'collection.maxSize': GAUGE,
322323
'collection.storageSize': GAUGE,
323324
'collection.nindexes': GAUGE,
325+
'collection.totalIndexSize': GAUGE,
326+
# collection latency stats
327+
'collection.reads.latency': GAUGE,
328+
'collection.reads.ops': RATE,
329+
'collection.writes.ops': RATE,
330+
'collection.writes.latency': GAUGE,
331+
'collection.commands.latency': GAUGE,
332+
'collection.commands.ops': RATE,
333+
'collection.transactions.latency': GAUGE,
334+
'collection.transactions.ops': RATE,
335+
# collection query exec stats
336+
'collection.collectionScans.total': GAUGE,
337+
'collection.collectionScans.nonTailable': GAUGE,
324338
}
325339

326340
"""

0 commit comments

Comments
 (0)