Send Teleport service check as a metric (#17441)

* update description 'cluster' -> 'instance'

* send count metric instead of service check

* raise on exception

* format

* fix integration tests

* remove obsolete field version in docker-compose.yml

* fix msg arg to self.count and add teleport_status tag to health.up metric

* fix integration tests typo

* fix e2e tests

* log error message on exception

* lint

* add teleport.health.up to metadata.csv

* add assertion for unreachable health state metric tag

* apply suggestion

Author: NouemanKHAL

teleport/datadog_checks/teleport/check.py

@@ -20,11 +20,18 @@ def __init__(self, name, init_config, instances):
 
     def check(self, _):
         try:
-            response = self.http.get("{}/healthz".format(self.diag_addr))
+            health_endpoint = f"{self.diag_addr}/healthz"
+            response = self.http.get(health_endpoint)
             response.raise_for_status()
-            self.service_check("health.up", self.OK)
+            self.count("health.up", 1, tags=["teleport_status:ok"])
         except Exception as e:
-            self.service_check("health.up", self.CRITICAL, message=str(e))
+            self.log.error(
+                "Cannot connect to Teleport HTTP diagnostic health endpoint '%s': %s.\nPlease make sure to enable Teleport's diagnostic HTTP endpoints.",  # noqa: E501
+                health_endpoint,
+                str(e),
+            )  # noqa: E501
+            self.count("health.up", 0, tags=["teleport_status:unreachable"])
+            raise
 
         super().check(_)
 

teleport/metadata.csv

@@ -128,3 +128,4 @@ teleport.db.messages_from_client.count,count,,,,Time to fetch TLS configuration
 teleport.db.messages_from_server.count,gauge,,,,Number of reverse SSH tunnels connected to the Teleport Proxy Service by Teleport instances,0,teleport,,
 teleport.db.method_call_count.count,gauge,,,,Number of outbound connections to leaf clusters,0,teleport,,
 teleport.db.method_call_latency_seconds.bucket,count,,,,Number of times a user exceeded their max concurrent ssh connections,0,teleport,,
+teleport.health.up,count,,,,Status of the Teleport instance,0,teleport,,

teleport/tests/docker/docker-compose.yaml

@@ -1,4 +1,3 @@
-version: '3'
 services:
   teleport:
     image: public.ecr.aws/gravitational/teleport:14.3

teleport/tests/test_common.py

@@ -11,11 +11,14 @@
 pytestmark = [pytest.mark.unit]
 
 
-def test_connect_exception(dd_run_check):
-    with pytest.raises(Exception):
+def test_connect_exception(dd_run_check, aggregator, caplog):
+    with pytest.raises(Exception, match="Failed to resolve 'invalid-hostname'"):
         check = TeleportCheck("teleport", {}, [BAD_HOSTNAME_INSTANCE])
         dd_run_check(check)
 
+    aggregator.assert_metric("teleport.health.up", value=0, count=1, tags=["teleport_status:unreachable"])
+    assert "Cannot connect to Teleport HTTP diagnostic health endpoint" in caplog.text
+
 
 def test_common_teleport_metrics(dd_run_check, aggregator, instance, mock_http_response, metrics_path):
     mock_http_response(file_path=metrics_path)

teleport/tests/test_e2e.py

@@ -4,8 +4,6 @@
 
 import pytest
 
-from datadog_checks.teleport import TeleportCheck
-
 from .common import COMMON_METRICS, INSTANCE
 
 pytestmark = pytest.mark.e2e
@@ -18,5 +16,5 @@
 
 def test_teleport_e2e(dd_agent_check):
     aggregator = dd_agent_check(CONFIG)
-    aggregator.assert_service_check('teleport.health.up', status=TeleportCheck.OK, count=1)
+    aggregator.assert_metric("teleport.health.up", value=1, count=1, tags=["teleport_status:ok"])
     aggregator.assert_metric(f"teleport.{COMMON_METRICS[0]}")

teleport/tests/test_integration.py

@@ -14,8 +14,7 @@
 def test_connect_ok(aggregator, instance, dd_run_check):
     check = TeleportCheck("teleport", {}, [instance])
     dd_run_check(check)
-    aggregator.assert_service_check("teleport.health.up", status=TeleportCheck.OK, count=1)
-    aggregator.assert_service_check("teleport.health.up", status=TeleportCheck.CRITICAL, count=0)
+    aggregator.assert_metric("teleport.health.up", value=1, count=1, tags=["teleport_status:ok"])
 
 
 def test_check_collects_teleport_common_metrics(aggregator, instance, dd_run_check):