[PLINT-394] Add support for swift component (#17640)

* [PLINT-394] Add support for swift component
### What does this PR do?
<!-- A brief description of the change being made with this pull request. -->
Adds support for the swift component in openstack, adding a service check, response time metric, and container count/bytes metric.

### Motivation
<!-- What inspired you to submit this pull request? -->

### Additional Notes
<!-- Anything else we should know when reviewing? -->

### Review checklist (to be filled by reviewers)

- [ ] Feature or bugfix MUST have appropriate tests (unit, integration, e2e)
- [ ] [Changelog entries](https://datadoghq.dev/integrations-core/guidelines/pr/#changelog-entries) must be created for modifications to shipped code
- [ ] Add the `qa/skip-qa` label if the PR doesn't need to be tested during QA.
- [ ] If you need to backport this PR to another branch, you can add the `backport/<branch-name>` label to the PR and it will automatically open a backport PR once this one is merged

* added changelog

Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com>

* updating swift license

Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com>

* reducing number of expected api calls due to different implementation of api_rest

Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com>

* changing swift port from 8080 to 6002

Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com>

* removing ?format=json from the default swift endpoint

Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com>

* Fixed error in test_response_time tests for swift component

* specifying the format=json parameter before the limit parameter

Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com>

* removing requests import

Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com>

* small refactor

* moved out logic for removing project_id in get_response_time

* fixed linter errors

* fixed linter error

* fixing test_response_time

Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com>

* removing debug statement

Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com>

* fixed test_containers_exception

Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com>

* improved test_containers_exception

Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com>

---------

Signed-off-by: rahulkaukuntla <rahul.kaukuntla@datadoghq.com>
Co-authored-by: Jose Manuel Almaza <josemanuel.almaza@datadoghq.com>

Author: rahulkaukuntla

openstack_controller/assets/configuration/spec.yaml

@@ -662,6 +662,13 @@ files:
                     properties:
                       - name: stacks
                         type: boolean
+              - name: swift
+                anyOf:
+                  - type: boolean
+                  - type: object
+                    properties:
+                      - name: containers
+                        type: boolean
             example:
               compute: false
         - name: projects

openstack_controller/assets/service_checks.json

@@ -164,5 +164,21 @@
             "region_id"
         ],
         "description": "Returns `CRITICAL` if the Agent is unable to query the HEAT API and `UNKNOWN` if endpoint is not found in the catalog. Returns `OK` otherwise."
+    },
+    {
+        "agent_version": "7.55.0",
+        "check": "openstack.swift.api.up",
+        "statuses": [
+            "ok",
+            "critical"
+        ],
+        "integration": "OpenStack",
+        "name": "OpenStack SWIFT API Up",
+        "groups": [
+            "host",
+            "keystone_server",
+            "region_id"
+        ],
+        "description": "Returns `CRITICAL` if the Agent is unable to query the SWIFT API and `UNKNOWN` if endpoint is not found in the catalog. Returns `OK` otherwise."
     }
 ]

openstack_controller/changelog.d/17640.added

@@ -0,0 +1 @@
+Added Swift component

openstack_controller/datadog_checks/openstack_controller/api/api_rest.py

@@ -2,7 +2,6 @@
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
 
-
 from datadog_checks.openstack_controller.api.api import Api
 from datadog_checks.openstack_controller.api.catalog import Catalog
 from datadog_checks.openstack_controller.components.component import Component
@@ -31,12 +30,13 @@ def has_admin_role(self):
     def component_in_catalog(self, component_types):
         return self._catalog.has_component(component_types)
 
-    def get_response_time(self, endpoint_types):
+    def get_response_time(self, endpoint_types, remove_project_id=True):
         endpoint = (
             self._catalog.get_endpoint_by_type(endpoint_types).replace(self._current_project_id, "")
-            if self._current_project_id
+            if self._current_project_id and remove_project_id
             else self._catalog.get_endpoint_by_type(endpoint_types)
         )
+        self.log.debug("getting response time for `%s`", endpoint)
         response = self.http.get(endpoint)
         response.raise_for_status()
         return response.elapsed.total_seconds() * 1000
@@ -199,13 +199,11 @@ def get_identity_limits(self):
         return response.json().get('limits', [])
 
     def get_block_storage_volumes(self, project_id):
-        params = {}
         return self.make_paginated_request(
             '{}/volumes/detail'.format(self._catalog.get_endpoint_by_type(Component.Types.BLOCK_STORAGE.value)),
             'volumes',
             'id',
             next_signifier='volumes_links',
-            params=params,
         )
 
     def get_block_storage_transfers(self, project_id):
@@ -218,13 +216,11 @@ def get_block_storage_transfers(self, project_id):
         return response.json().get('transfers', {})
 
     def get_block_storage_snapshots(self, project_id):
-        params = {}
         return self.make_paginated_request(
             '{}/snapshots/detail'.format(self._catalog.get_endpoint_by_type(Component.Types.BLOCK_STORAGE.value)),
             'snapshots',
             'id',
             next_signifier='snapshots_links',
-            params=params,
         )
 
     def get_block_storage_pools(self, project_id):
@@ -358,27 +354,25 @@ def make_request(url, params):
 
         if self.config.paginated_limit is None:
             response_json = make_request(url, params)
-            objects = response_json.get(resource_name, [])
-            return objects
+            return response_json if resource_name is None else response_json.get(resource_name, [])
 
+        params['limit'] = self.config.paginated_limit
         while True:
             self.log.debug(
                 "making paginated request [limit=%s, marker=%s]",
                 self.config.paginated_limit,
                 marker,
             )
-
-            params['limit'] = self.config.paginated_limit
             if marker is not None:
                 params['marker'] = marker
 
             response_json = make_request(url, params)
-            resources = response_json.get(resource_name, [])
+            resources = response_json if resource_name is None else response_json.get(resource_name, [])
             if len(resources) > 0:
                 last_item = resources[-1]
                 item_list.extend(resources)
 
-                if next_signifier == '{}_links'.format(resource_name):
+                if next_signifier == f'{resource_name}_links':
                     has_next_link = False
                     links = response_json.get(next_signifier, [])
                     for link in links:
@@ -389,7 +383,11 @@ def make_request(url, params):
                     if not has_next_link:
                         break
                 else:
-                    next_item = response_json.get(next_signifier)
+                    next_item = (
+                        response_json[-1].get(next_signifier)
+                        if resource_name is None
+                        else response_json.get(next_signifier)
+                    )
                     if next_item is None:
                         break
 
@@ -607,3 +605,13 @@ def get_heat_stacks(self, project_id):
             'id',
             next_signifier='links',
         )
+
+    def get_swift_containers(self, account_id):
+        params = {'format': 'json'}
+        return self.make_paginated_request(
+            '{}'.format(self._catalog.get_endpoint_by_type(Component.Types.SWIFT.value)),
+            None,
+            'name',
+            next_signifier='name',
+            params=params,
+        )

openstack_controller/datadog_checks/openstack_controller/api/api_sdk.py

@@ -115,9 +115,11 @@ def authorize_project(self, project_id):
         self.connection.authorize()
         self.http.options['headers']['X-Auth-Token'] = self.connection.session.auth.get_token(self.connection.session)
 
-    def get_response_time(self, endpoint_types):
+    def get_response_time(self, endpoint_types, remove_project_id=True):
         endpoint = self._catalog.get_endpoint_by_type(endpoint_types)
-        endpoint = endpoint.replace(self._access.project_id, "") if self._access.project_id else endpoint
+        endpoint = (
+            endpoint.replace(self._access.project_id, "") if self._access.project_id and remove_project_id else endpoint
+        )
         response = self.http.get(endpoint)
         response.raise_for_status()
         return response.elapsed.total_seconds() * 1000
@@ -394,3 +396,11 @@ def get_heat_stacks(self, project_id):
                 self.connection.heat.stacks, project_id=project_id, limit=self.config.paginated_limit
             )
         ]
+
+    def get_swift_containers(self, account_id):
+        return [
+            container.to_dict(original_names=True)
+            for container in self.call_paginated_api(
+                self.connection.swift.containers, account_id=account_id, limit=self.config.paginated_limit
+            )
+        ]

openstack_controller/datadog_checks/openstack_controller/components/component.py

@@ -49,6 +49,7 @@ class Id(str, Enum):
         LOAD_BALANCER = 'load-balancer'
         IMAGE = 'image'
         HEAT = 'heat'
+        SWIFT = 'swift'
 
     @unique
     class Types(list, Enum):
@@ -60,6 +61,7 @@ class Types(list, Enum):
         LOAD_BALANCER = ['load-balancer']
         IMAGE = ['image']
         HEAT = ['orchestration']
+        SWIFT = ['object-store']
 
     def http_error(report_service_check=False):
         def decorator_http_error(func):

openstack_controller/datadog_checks/openstack_controller/components/swift.py

@@ -0,0 +1,51 @@
+# (C) Datadog, Inc. 2024-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+
+
+from datadog_checks.openstack_controller.components.component import Component
+from datadog_checks.openstack_controller.metrics import (
+    SWIFT_CONTAINER_COUNT,
+    SWIFT_CONTAINER_METRICS,
+    SWIFT_CONTAINER_PREFIX,
+    SWIFT_CONTAINER_TAGS,
+    SWIFT_RESPONSE_TIME,
+    SWIFT_SERVICE_CHECK,
+    get_metrics_and_tags,
+)
+
+
+class Swift(Component):
+    ID = Component.Id.SWIFT
+    TYPES = Component.Types.SWIFT
+    SERVICE_CHECK = SWIFT_SERVICE_CHECK
+
+    def __init__(self, check):
+        super(Swift, self).__init__(check)
+
+    @Component.register_global_metrics(ID)
+    @Component.http_error(report_service_check=True)
+    def _report_response_time(self, global_components_config, tags):
+        self.check.log.debug("reporting `%s` response time", Swift.ID.value)
+        response_time = self.check.api.get_response_time(Swift.TYPES.value, remove_project_id=False)
+        self.check.log.debug("`%s` response time: %s", Swift.ID.value, response_time)
+        self.check.gauge(SWIFT_RESPONSE_TIME, response_time, tags=tags)
+
+    @Component.register_project_metrics(ID)
+    @Component.http_error()
+    def _report_containers(self, project_id, tags, config):
+        report_containers = config.get('containers', True)
+        if report_containers:
+            data = self.check.api.get_swift_containers(project_id)
+            self.check.log.debug("containers: %s", data)
+            for item in data:
+                container = get_metrics_and_tags(
+                    item,
+                    tags=SWIFT_CONTAINER_TAGS,
+                    prefix=SWIFT_CONTAINER_PREFIX,
+                    metrics=SWIFT_CONTAINER_METRICS,
+                )
+                self.check.log.debug("container: %s", container)
+                self.check.gauge(SWIFT_CONTAINER_COUNT, 1, tags=tags + container['tags'])
+                for metric, value in container['metrics'].items():
+                    self.check.gauge(metric, value, tags=tags + container['tags'])

openstack_controller/datadog_checks/openstack_controller/config_models/instance.py

@@ -327,6 +327,14 @@ class NetworkItem(BaseModel):
     quotas: Optional[bool] = None
 
 
+class SwiftItem(BaseModel):
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        frozen=True,
+    )
+    containers: Optional[bool] = None
+
+
 class Components(BaseModel):
     model_config = ConfigDict(
         arbitrary_types_allowed=True,
@@ -340,6 +348,7 @@ class Components(BaseModel):
     image: Optional[Union[bool, ImageItem]] = None
     load_balancer: Optional[Union[bool, LoadBalancerItem]] = Field(None, alias='load-balancer')
     network: Optional[Union[bool, NetworkItem]] = None
+    swift: Optional[Union[bool, SwiftItem]] = None
 
 
 class MetricPatterns(BaseModel):

openstack_controller/datadog_checks/openstack_controller/metrics.py

@@ -645,6 +645,18 @@
     'stack_status': 'stack_status',
 }
 
+SWIFT_METRICS_PREFIX = "openstack.swift"
+SWIFT_SERVICE_CHECK = f"{SWIFT_METRICS_PREFIX}.api.up"
+SWIFT_RESPONSE_TIME = f"{SWIFT_METRICS_PREFIX}.response_time"
+SWIFT_CONTAINER_PREFIX = f"{SWIFT_METRICS_PREFIX}.container"
+SWIFT_CONTAINER_COUNT = f"{SWIFT_CONTAINER_PREFIX}.count"
+SWIFT_CONTAINER_METRICS = {
+    f"{SWIFT_CONTAINER_PREFIX}.bytes": {},
+}
+SWIFT_CONTAINER_TAGS = {
+    'name': 'container_name',
+}
+
 
 def is_interface_metric(label):
     return any(seg in label for seg in ['_rx', '_tx'])

openstack_controller/datadog_checks/openstack_controller/openstack_controller.py

@@ -16,6 +16,7 @@
 from datadog_checks.openstack_controller.components.image import Image
 from datadog_checks.openstack_controller.components.load_balancer import LoadBalancer
 from datadog_checks.openstack_controller.components.network import Network
+from datadog_checks.openstack_controller.components.swift import Swift
 from datadog_checks.openstack_controller.config import OpenstackConfig, normalize_discover_config_include
 
 from .config_models import ConfigMixin
@@ -53,6 +54,7 @@ def init(self):
             LoadBalancer(self),
             Image(self),
             Heat(self),
+            Swift(self),
         ]
         self.projects_discovery = None
         if self.config.projects:

openstack_controller/metadata.csv

@@ -272,3 +272,6 @@ openstack.octavia.quota.loadbalancer,gauge,,,,The configured load balancer quota
 openstack.octavia.quota.member,gauge,,,,The configured member quota limit,0,openstack_controller,octavia quota member,
 openstack.octavia.quota.pool,gauge,,,,The configured pool quota limit,0,openstack_controller,octavia quota pool,
 openstack.octavia.response_time,gauge,,millisecond,,Duration that an HTTP request takes to complete when making a request to octavia endpoint,0,openstack_controller,octavia response time,
+openstack.swift.container.bytes,gauge,,,,Number of bytes in a swift container,0,openstack_controller,swift container bytes,
+openstack.swift.container.count,gauge,,,,"Number of swift containers",0,openstack_controller,swift container ct,
+openstack.swift.response_time,gauge,,millisecond,,"Duration that an HTTP request takes to complete when making a request to swift endpoint",0,openstack_controller,swift response time,

openstack_controller/tests/conftest.py

@@ -188,7 +188,7 @@ def method(method, url, file='response', headers=None, params=None):
 def mock_http_call(mock_responses):
     def call(method, url, file='response', headers=None, params=None):
         response = mock_responses(method, url, file=file, headers=headers, params=params)
-        if response:
+        if response is not None:
             return response
         http_response = requests.models.Response()
         http_response.status_code = 404
@@ -921,6 +921,26 @@ def stacks(project_id, limit=None):
     return mock.MagicMock(stacks=mock.MagicMock(side_effect=stacks))
 
 
+@pytest.fixture
+def connection_swift(request, mock_responses):
+    param = request.param if hasattr(request, 'param') and request.param is not None else {}
+    http_error = param.get('http_error')
+
+    def containers(account_id, limit=None):
+        if http_error and 'containers' in http_error and account_id in http_error['containers']:
+            raise requests.exceptions.HTTPError(response=http_error['containers'])
+        return [
+            mock.MagicMock(
+                to_dict=mock.MagicMock(
+                    return_value=node,
+                )
+            )
+            for node in mock_responses('GET', f'/v1/AUTH_{account_id}/format=json')
+        ]
+
+    return mock.MagicMock(containers=mock.MagicMock(side_effect=containers))
+
+
 @pytest.fixture
 def openstack_connection(
     openstack_session,
@@ -933,6 +953,7 @@ def openstack_connection(
     connection_load_balancer,
     connection_image,
     connection_heat,
+    connection_swift,
 ):
     def connection(cloud, session, region_name):
         return mock.MagicMock(
@@ -946,6 +967,7 @@ def connection(cloud, session, region_name):
             load_balancer=connection_load_balancer,
             image=connection_image,
             heat=connection_heat,
+            swift=connection_swift,
         )
 
     with mock.patch('openstack.connection.Connection', side_effect=connection) as mock_connection:

openstack_controller/tests/docker/docker-compose.yaml

@@ -1,4 +1,3 @@
-version: "3"
 
 services:
   openstack-keystone:
@@ -66,3 +65,11 @@ services:
       - ./etc/caddy/glance:/etc/caddy/
     ports:
       - "8004:80"
+  openstack-swift:
+    image: caddy:2.6.2-alpine
+    container_name: openstack-swift
+    volumes:
+      - ../fixtures:/usr/share/caddy
+      - ./etc/caddy/glance:/etc/caddy/
+    ports:
+      - "6002:80"

openstack_controller/tests/fixtures/GET/v1/AUTH_1e6e233e637d4d55a50a62b63398ad15/format=json/limit=1/marker=container_1/response.json

@@ -0,0 +1,8 @@
+[
+    {
+        "name": "container_2",
+        "count": 0,
+        "bytes": 0,
+        "last_modified": "2024-05-22T14:40:36.172670"
+    }
+]

openstack_controller/tests/fixtures/GET/v1/AUTH_1e6e233e637d4d55a50a62b63398ad15/format=json/limit=1/marker=container_2/response.json

@@ -0,0 +1 @@
+[]

openstack_controller/tests/fixtures/GET/v1/AUTH_1e6e233e637d4d55a50a62b63398ad15/format=json/limit=1/response.json

@@ -0,0 +1,8 @@
+[
+    {
+        "name": "container_1",
+        "count": 1,
+        "bytes": 490,
+        "last_modified": "2024-05-22T13:23:23.011420"
+    }
+]