Add metric to track seconds since last backfill cycle (#20165)

* add metric to track seconds since last backfill

* changelog

* Update check.py

* lint

Author: steveny91

slurm/changelog.d/20165.added

@@ -0,0 +1 @@
+Add metric to track seconds since last backfill cycle from sdiag

slurm/datadog_checks/slurm/check.py

@@ -2,7 +2,9 @@
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
 import os
+import re
 import subprocess
+import time
 from datetime import timedelta
 
 from datadog_checks.base import AgentCheck, is_affirmative
@@ -311,6 +313,7 @@ def process_sdiag(self, output):
             # we see the 'Backfilling stats' line.
             current_map = SDIAG_MAP['backfill_stats'] if backfill_section else SDIAG_MAP['main_stats']
 
+            # Try to match known metrics
             for metric, pattern in current_map.items():
                 if pattern in line:
                     try:
@@ -321,6 +324,17 @@ def process_sdiag(self, output):
                         continue
                     break
 
+            if 'Last cycle when' in line:
+                try:
+                    match = re.search(r'\((\d+)\)', line)
+                    if match:
+                        last_cycle_epoch = int(match.group(1))
+                        now = int(time.time())
+                        diff = now - last_cycle_epoch
+                        self.gauge('sdiag.last_cycle_seconds_ago', diff, tags=self.tags)
+                except Exception as e:
+                    self.log.debug("Failed to parse last cycle epoch from line '%s': %s", line, e)
+
         for name, value in metrics.items():
             self.gauge(f'sdiag.{name}', value, tags=self.tags)
 

slurm/metadata.csv

@@ -54,6 +54,7 @@ slurm.sdiag.jobs_running,gauge,,job,,Number of jobs running since last reset.,1,
 slurm.sdiag.jobs_started,gauge,,job,,Number of jobs started since last reset.,1,slurm,slurm_sdiag_jobs_started,,
 slurm.sdiag.jobs_submitted,gauge,,job,,Number of jobs submitted since last reset.,1,slurm,slurm_sdiag_jobs_submitted,,
 slurm.sdiag.last_cycle,gauge,,microsecond,,Time in microseconds for last scheduling cycle.,1,slurm,slurm_sdiag_last_cycle,,
+slurm.sdiag.last_cycle_seconds_ago,gauge,,second,,Time in seconds since the last scheduling cycle.,1,slurm,slurm_sdiag_last_cycle_seconds_ago,,
 slurm.sdiag.last_queue_length,gauge,,job,,Length of jobs pending queue.,1,slurm,slurm_sdiag_last_queue_length,,
 slurm.sdiag.max_cycle,gauge,,microsecond,,Maximum time in microseconds for any scheduling cycle since last reset.,1,slurm,slurm_sdiag_max_cycle,,
 slurm.sdiag.mean_cycle,gauge,,microsecond,,Mean time in microseconds for all scheduling cycles since last reset.,1,slurm,slurm_sdiag_mean_cycle,,

slurm/tests/common.py

@@ -1138,6 +1138,11 @@ def mock_output(filename):
         {'name': 'slurm.sdiag.backfill.depth_mean_try_depth', 'value': 27, 'tags': []},
         {'name': 'slurm.sdiag.backfill.queue_length_mean', 'value': 28, 'tags': []},
         {'name': 'slurm.sdiag.backfill.mean_table_size', 'value': 29, 'tags': []},
+        {
+            'name': 'slurm.sdiag.last_cycle_seconds_ago',
+            'value': 1000,
+            'tags': [],
+        },  # mocked to be 1000 seconds in the test
     ]
 }
 

slurm/tests/test_unit.py

@@ -118,7 +118,17 @@ def test_slurm_binary_processing(mock_get_subprocess_output, instance, aggregato
     else:
         mock_get_subprocess_output.side_effect = [mock_output_main]
 
-    check.check(None)
+    # Patch time.time only for sdiag to make the test deterministic
+    if binary == 'sdiag':
+        from unittest.mock import patch
+
+        # The epoch in sdiag.txt is 1726207912, mocking current time to 1726208912 (diff = 1000)
+        with patch('datadog_checks.slurm.check.time') as mock_time:
+            mock_time.time.return_value = 1726208912
+            check.check(None)
+    else:
+        check.check(None)
+
     if binary == 'sacct':
         # This one doesn't collect anything on the first run. It only collects on the second run.
         check.check(None)