Introduce a new foundationdb.processes_per_role gauge tagged by role/process class (#19706)

Author: jon-signal

foundationdb/changelog.d/19706.added

@@ -0,0 +1 @@
+Introduce a `foundationdb.processes_per_role` gauge tagged by process role/class

foundationdb/datadog_checks/foundationdb/check.py

@@ -233,20 +233,47 @@ def check_metrics(self, status):
             )
 
             role_counts = {}
+            process_counts_by_tag_set = {}
 
             for process_key in processes:
                 process = processes[process_key]
 
                 self.report_process(process)
+
+                process_tags = tags.copy()
+
+                if "class_type" in process:
+                    process_tags.append("fdb_process_class:" + process["class_type"])
+
                 if "roles" in process:
                     for role in process["roles"]:
                         if "role" in role:
                             rolename = role["role"]
+                            process_tags.append("fdb_role:" + rolename)
                             if rolename in role_counts:
                                 role_counts[rolename] += 1
                             else:
                                 role_counts[rolename] = 1
 
+                process_tags.sort()
+
+                # Packing and unpacking tags into sorted, comma-separated strings is a little hacky, but allows us to
+                # keep a map of role sets to counts; lists/sets aren't hashable and won't work as dictionary keys.
+                tags_string = ",".join(process_tags)
+
+                if tags_string in process_counts_by_tag_set:
+                    process_counts_by_tag_set[tags_string] += 1
+                else:
+                    process_counts_by_tag_set[tags_string] = 1
+
+            for tags_string in process_counts_by_tag_set:
+                process_tags = tags_string.split(",")
+                self.gauge(
+                    "processes_per_role",
+                    process_counts_by_tag_set[tags_string],
+                    process_tags if process_tags else None,
+                )
+
             for role in role_counts:
                 self.gauge("processes_per_role." + role, role_counts[role], tags)
 

foundationdb/metadata.csv

@@ -94,6 +94,7 @@ foundationdb.process.role.stored_bytes,gauge,,byte,,,0,foundationdb,stored bytes
 foundationdb.process.role.total_queries.counter,count,,query,,,0,foundationdb,total queries,
 foundationdb.process.role.total_queries.hz,gauge,,query,,,0,foundationdb,total queries,
 foundationdb.processes,gauge,,process,,,0,foundationdb,processes,
+foundationdb.processes_per_role,gauge,,process,,,0,foundationdb,processes per role/process class,
 foundationdb.processes_per_role.cluster_controller,gauge,,process,,,0,foundationdb,processes per cluster controller,
 foundationdb.processes_per_role.coordinator,gauge,,process,,,0,foundationdb,processes per coordinator,
 foundationdb.processes_per_role.data_distributor,gauge,,process,,,0,foundationdb,processes per data distributor,

foundationdb/tests/common.py

@@ -189,6 +189,7 @@
     "foundationdb.process.role.read_latency_statistics.p90",
     "foundationdb.process.role.read_latency_statistics.p99",
     "foundationdb.process.role.queue_length",
+    "foundationdb.processes_per_role",
     "foundationdb.processes_per_role.cluster_controller",
     "foundationdb.processes_per_role.coordinator",
     "foundationdb.processes_per_role.data_distributor",

foundationdb/tests/test_foundationdb.py

@@ -63,6 +63,18 @@ def test_full(aggregator, instance):
         aggregator.assert_metrics_using_metadata(get_metadata_metrics())
         aggregator.assert_service_check("foundationdb.can_connect", AgentCheck.OK)
 
+        aggregator.assert_metric_has_tag("foundationdb.processes_per_role", "fdb_process_class:unset")
+
+        aggregator.assert_metric_has_tag("foundationdb.processes_per_role", "fdb_role:master")
+        aggregator.assert_metric_has_tag("foundationdb.processes_per_role", "fdb_role:cluster_controller")
+        aggregator.assert_metric_has_tag("foundationdb.processes_per_role", "fdb_role:data_distributor")
+        aggregator.assert_metric_has_tag("foundationdb.processes_per_role", "fdb_role:ratekeeper")
+        aggregator.assert_metric_has_tag("foundationdb.processes_per_role", "fdb_role:coordinator")
+        aggregator.assert_metric_has_tag("foundationdb.processes_per_role", "fdb_role:proxy")
+        aggregator.assert_metric_has_tag("foundationdb.processes_per_role", "fdb_role:log")
+        aggregator.assert_metric_has_tag("foundationdb.processes_per_role", "fdb_role:storage")
+        aggregator.assert_metric_has_tag("foundationdb.processes_per_role", "fdb_role:resolver")
+
 
 @pytest.mark.skipif(PROTOCOL == 'tls', reason="Non-TLS FoundationDB cluster only.")
 @pytest.mark.usefixtures("dd_environment")