Merge branch 'master' into sy/ddev-bump-dcd

Author: steveny91

.in-toto/tag.ec45eb9d.link

@@ -2,6 +2,12 @@
 
 <!-- towncrier release notes start -->
 
+## 32.1.1 / 2024-05-09
+
+***Fixed***:
+
+* Fix trailing `,` in manifest.json generation template ([#17538](https://github.com/DataDog/integrations-core/pull/17538))
+
 ## 32.1.0 / 2024-05-03
 
 ***Added***:

datadog_checks_dev/CHANGELOG.md

@@ -1,4 +1,4 @@
 # (C) Datadog, Inc. 2018-present
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
-__version__ = '32.1.0'
+__version__ = '32.1.1'

datadog_checks_dev/changelog.d/17538.fixed

@@ -0,0 +1 @@
+Add sample_tags to metadata validation

datadog_checks_dev/datadog_checks/dev/__about__.py

@@ -130,12 +130,17 @@ def metadata(app: Application, integrations: tuple[str, ...], check_duplicates:
 
                     error_message += f"{current_check.name}:{line} Invalid column {invalid_headers}.\n"
 
-                missing_headers = metadata_utils.ALL_HEADERS.difference(all_keys)
+                missing_headers = metadata_utils.HEADERS_TO_CHECK.difference(all_keys)
                 if missing_headers:
                     errors = True
 
                     error_message += f"{current_check.name}:{line} Missing columns {missing_headers}.\n"
-                continue
+
+                if errors:
+                    # There's now an optional sample_tag column that isn't added yet to the existing metadata.csv
+                    # all_keys will not be same as ALL_HEADERS. But since that sample_tag column is optional and not
+                    # inside HEADERS_TO_CHECK, we should only continue if there's an invalid header or missing_header.
+                    continue
 
             # check duplicate metric name
             duplicate_metric_name = check_duplicate_values(

ddev/changelog.d/17521.added

@@ -5,9 +5,18 @@
 
 REQUIRED_HEADERS = {'metric_name', 'metric_type', 'orientation', 'integration'}
 
-OPTIONAL_HEADERS = {'description', 'interval', 'unit_name', 'per_unit_name', 'short_name', 'curated_metric'}
+OPTIONAL_HEADERS = {
+    'description',
+    'interval',
+    'unit_name',
+    'per_unit_name',
+    'short_name',
+    'curated_metric',
+}
 
-ALL_HEADERS = REQUIRED_HEADERS | OPTIONAL_HEADERS
+EXPERIMENTAL_HEADER = {"sample_tags"}
+ALL_HEADERS = REQUIRED_HEADERS | OPTIONAL_HEADERS | EXPERIMENTAL_HEADER
+HEADERS_TO_CHECK = REQUIRED_HEADERS | OPTIONAL_HEADERS
 
 ORDERED_HEADERS = [
     "metric_name",
@@ -20,6 +29,7 @@
     "integration",
     "short_name",
     "curated_metric",
+    "sample_tags",
 ]
 
 VALID_METRIC_TYPE = {'count', 'gauge', 'rate'}

ddev/src/ddev/cli/validate/metadata.py

@@ -715,6 +715,58 @@ def test_metrics_ordered(fake_repo, ddev, helpers):
     )
 
 
+def test_passing_with_experimental_column(fake_repo, ddev, helpers):
+    # Testing to ensure that experimental header sample_tags is allowed
+    write_file(
+        fake_repo.path / "metadata_integration",
+        'metadata.csv',
+        """metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric,sample_tags
+metadata_integration.metric_a,gauge,,,,My metric A,0,metadata_integration,,,
+metadata_integration.metric_b,gauge,,,,My metric B,0,metadata_integration,,,
+""",
+    )
+
+    result = ddev('validate', 'metadata', 'metadata_integration')
+
+    assert result.exit_code == 0, result.output
+    assert helpers.remove_trailing_spaces(result.output) == helpers.dedent(
+        """
+        Metrics validation
+
+        Passed: 1
+        """
+    )
+
+
+def test_passing_invalid_experimental_column(fake_repo, ddev, helpers):
+    # Testing to ensure that experimental header sample_tags is allowed. But if other tags are added,
+    # it will be flagged as an error
+    write_file(
+        fake_repo.path / "metadata_integration",
+        'metadata.csv',
+        """metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric,sample_tags,foo
+metadata_integration.metric_a,gauge,,,,My metric A,0,metadata_integration,,,,
+metadata_integration.metric_b,gauge,,,,My metric B,0,metadata_integration,,,,
+""",
+    )
+    outfile = os.path.join('metadata_integration', 'metadata.csv')
+    result = ddev('validate', 'metadata', 'metadata_integration')
+
+    assert result.exit_code == 1, result.output
+    assert helpers.remove_trailing_spaces(result.output) == helpers.dedent(
+        f"""
+        Metrics validation
+        └── metadata_integration
+            └── {outfile}
+
+                metadata_integration:2 Invalid column {{'foo'}}.
+                metadata_integration:3 Invalid column {{'foo'}}.
+
+        Errors: 1
+        """
+    )
+
+
 def test_metrics_not_ordered(fake_repo, ddev, helpers):
     outfile = os.path.join('metadata_integration', 'metadata.csv')
     result = ddev('validate', 'metadata', 'metadata_integration')

ddev/src/ddev/cli/validate/metadata_utils.py

@@ -42,6 +42,14 @@ files:
           value:
             type: boolean
             example: false
+        - name: use_configured_hostname
+          description: |
+            If true, the check will use the configured `host` parameter for ESXi hostnames instead of the Host name.
+            You may need to use this if you install both the vSphere check and ESXi check to avoid duplicate entries
+            for hosts in the web application UI.
+          value:
+            type: boolean
+            example: true
         - name: collect_per_instance_filters
           description: |
             Use this option to collect metrics tagged with instance values.
@@ -139,8 +147,8 @@ files:
           description: |
             Use this option to send a subset of host tags as metric tags.
             The ESXi integration collects tags for every ESXi host or VM in your environment.
-            Those tags can be `esxi_url`, esxi_type`, `esxi_host`, `esxi_folder`, `esxi_cluster`
-            `esxi_compute`, `esxi_datacenter`, `esxi_datastore` and are attached to the ESXi metrics by default.
+            Those tags can be `esxi_url`, esxi_type`, `esxi_host`, and `esxi_datastore`
+            and are attached to the ESXi metrics by default.
             By default these tags are submitted as host tags, but you can submit them as metric tags instead by using
             this configuration option, see: https://docs.datadoghq.com/tagging/using_tags/
             You will lose the ability to filter your hosts on the tags you specify but they will appear faster

ddev/tests/cli/validate/test_metrics.py

@@ -0,0 +1 @@
+Add option to use configured host as hostname.

esxi/assets/configuration/spec.yaml

@@ -0,0 +1 @@
+Remove invalid host tags

esxi/changelog.d/17544.added

@@ -44,7 +44,8 @@ def __init__(self, name, init_config, instances):
         self.host = self.instance.get("host")
         self.username = self.instance.get("username")
         self.password = self.instance.get("password")
-        self.use_guest_hostname = self.instance.get("use_guest_hostname", False)
+        self.use_guest_hostname = is_affirmative(self.instance.get("use_guest_hostname", False))
+        self.use_configured_hostname = is_affirmative(self.instance.get("use_configured_hostname", False))
         self.excluded_host_tags = self._validate_excluded_host_tags(self.instance.get("excluded_host_tags", []))
         self.collect_per_instance_filters = self._parse_metric_regex_filters(
             self.instance.get("collect_per_instance_filters", {})
@@ -62,8 +63,7 @@ def _validate_excluded_host_tags(self, excluded_host_tags):
             if excluded_host_tag not in AVAILABLE_HOST_TAGS:
                 self.log.warning(
                     "Unknown host tag `%s` cannot be excluded. Available host tags are: "
-                    "`esxi_url`, `esxi_type`, `esxi_host`, `esxi_folder`, `esxi_cluster` "
-                    "`esxi_compute`, `esxi_datacenter`, and `esxi_datastore`",
+                    "`esxi_url`, `esxi_type`, `esxi_host`, `esxi_compute`, and `esxi_datastore`",
                     excluded_host_tag,
                 )
             else:
@@ -238,7 +238,7 @@ def get_available_metric_ids_for_entity(self, entity):
         metric_ids = [vim.PerformanceManager.MetricId(counterId=counter, instance="") for counter in counter_ids]
         return counter_keys_and_names, metric_ids
 
-    def collect_metrics_for_entity(self, metric_ids, counter_keys_and_names, entity, entity_name, metric_tags):
+    def collect_metrics_for_entity(self, metric_ids, counter_keys_and_names, entity, hostname, metric_tags):
         resource_type = type(entity)
         resource_name = RESOURCE_TYPE_TO_NAME[resource_type]
         for metric_id in metric_ids:
@@ -297,7 +297,7 @@ def collect_metrics_for_entity(self, metric_ids, counter_keys_and_names, entity,
                     self.log.debug(
                         "Skipping metric %s for %s because no value was returned by the %s",
                         metric_name,
-                        entity_name,
+                        hostname,
                         resource_name,
                     )
                     continue
@@ -308,7 +308,7 @@ def collect_metrics_for_entity(self, metric_ids, counter_keys_and_names, entity,
                         "Skipping metric %s for %s, because the value returned by the %s"
                         " is negative (i.e. the metric is not yet available). values: %s",
                         metric_name,
-                        entity_name,
+                        hostname,
                         resource_name,
                         list(metric_result.value),
                     )
@@ -321,10 +321,10 @@ def collect_metrics_for_entity(self, metric_ids, counter_keys_and_names, entity,
                         "Submit metric: name=`%s`, value=`%s`, hostname=`%s`, tags=`%s`",
                         metric_name,
                         most_recent_val,
-                        entity_name,
+                        hostname,
                         all_tags,
                     )
-                    self.gauge(metric_name, most_recent_val, hostname=entity_name, tags=all_tags)
+                    self.gauge(metric_name, most_recent_val, hostname=hostname, tags=all_tags)
 
     def set_version_metadata(self):
         esxi_version = self.content.about.version
@@ -388,6 +388,8 @@ def check(self, _):
         }
 
         for resource_obj, resource_props in all_resources_with_metrics.items():
+            resource_type = type(resource_obj)
+            resource_type_name = RESOURCE_TYPE_TO_NAME[resource_type]
 
             if not is_resource_collected_by_filters(resource_obj, all_resources_with_metrics, self.resource_filters):
                 self.log.debug(
@@ -397,16 +399,18 @@ def check(self, _):
 
             hostname = resource_props.get("name")
 
-            resource_type = RESOURCE_TYPE_TO_NAME[type(resource_obj)]
-            if resource_type == "vm" and self.use_guest_hostname:
+            if resource_type == VM_RESOURCE and self.use_guest_hostname:
                 hostname = resource_props.get("guest.hostName", hostname)
 
+            if resource_type == HOST_RESOURCE and self.use_configured_hostname:
+                hostname = self.host
+
             self.log.debug("Collect metrics and host tags for hostname: %s, object: %s", hostname, resource_obj)
 
             tags = []
             parent = resource_props.get('parent')
 
-            if resource_type == "vm":
+            if resource_type == VM_RESOURCE:
                 runtime_host = resource_props.get('runtime.host')
                 runtime_host_props = {}
                 if runtime_host:
@@ -418,19 +422,14 @@ def check(self, _):
                 runtime_hostname = to_string(runtime_host_props.get("name", "unknown"))
                 tags.append('esxi_host:{}'.format(runtime_hostname))
 
-                if runtime_host is not None:
-                    tags.extend(
-                        get_tags_recursively(
-                            runtime_host,
-                            resource_map,
-                            include_only=['esxi_cluster'],
-                        )
-                    )
-
             if parent is not None:
-                tags.extend(get_tags_recursively(parent, resource_map))
+                tags.extend(
+                    get_tags_recursively(
+                        parent, resource_map, include_only=["esxi_compute", "esxi_host", "esxi_datastore"]
+                    )
+                )
 
-            tags.append('esxi_type:{}'.format(resource_type))
+            tags.append('esxi_type:{}'.format(resource_type_name))
 
             metric_tags = self.tags
             if self.excluded_host_tags:
@@ -444,7 +443,7 @@ def check(self, _):
             else:
                 self.log.debug("No host name found for %s; skipping external tag submission", resource_obj)
 
-            self.count(f"{resource_type}.count", 1, tags=tags, hostname=None)
+            self.count(f"{resource_type_name}.count", 1, tags=tags, hostname=None)
 
             counter_keys_and_names, metric_ids = self.get_available_metric_ids_for_entity(resource_obj)
             self.collect_metrics_for_entity(metric_ids, counter_keys_and_names, resource_obj, hostname, metric_tags)

esxi/changelog.d/17549.fixed

@@ -28,5 +28,9 @@ def instance_ssl_verify():
     return True
 
 
+def instance_use_configured_hostname():
+    return True
+
+
 def instance_use_guest_hostname():
     return False

esxi/datadog_checks/esxi/check.py

@@ -78,6 +78,7 @@ class InstanceConfig(BaseModel):
     ssl_capath: Optional[str] = None
     ssl_verify: Optional[bool] = None
     tags: Optional[tuple[str, ...]] = None
+    use_configured_hostname: Optional[bool] = None
     use_guest_hostname: Optional[bool] = None
     username: str
 

esxi/datadog_checks/esxi/config_models/defaults.py

@@ -61,10 +61,7 @@
     "esxi_url",
     "esxi_type",
     "esxi_host",
-    "esxi_folder",
-    "esxi_cluster",
     "esxi_compute",
-    "esxi_datacenter",
     "esxi_datastore",
 ]
 

esxi/datadog_checks/esxi/config_models/instance.py

@@ -39,6 +39,13 @@ instances:
     #
     # use_guest_hostname: false
 
+    ## @param use_configured_hostname - boolean - optional - default: true
+    ## If true, the check will use the configured `host` parameter for ESXi hostnames instead of the Host name.
+    ## You may need to use this if you install both the vSphere check and ESXi check to avoid duplicate entries
+    ## for hosts in the web application UI.
+    #
+    # use_configured_hostname: true
+
     ## @param collect_per_instance_filters - mapping - optional
     ## Use this option to collect metrics tagged with instance values.
     ## Some ESXi metrics can be tagged with instance values.
@@ -101,8 +108,8 @@ instances:
     ## @param excluded_host_tags - list of strings - optional - default: []
     ## Use this option to send a subset of host tags as metric tags.
     ## The ESXi integration collects tags for every ESXi host or VM in your environment.
-    ## Those tags can be `esxi_url`, esxi_type`, `esxi_host`, `esxi_folder`, `esxi_cluster`
-    ## `esxi_compute`, `esxi_datacenter`, `esxi_datastore` and are attached to the ESXi metrics by default.
+    ## Those tags can be `esxi_url`, esxi_type`, `esxi_host`, and `esxi_datastore`
+    ## and are attached to the ESXi metrics by default.
     ## By default these tags are submitted as host tags, but you can submit them as metric tags instead by using
     ## this configuration option, see: https://docs.datadoghq.com/tagging/using_tags/
     ## You will lose the ability to filter your hosts on the tags you specify but they will appear faster

esxi/datadog_checks/esxi/constants.py

@@ -85,9 +85,6 @@ def test_vcsim_external_host_tags(vcsim_instance, datadog_agent, dd_run_check):
         {
             'esxi': [
                 'esxi_compute:localhost.localdomain',
-                'esxi_datacenter:ha-datacenter',
-                'esxi_folder:ha-folder-root',
-                'esxi_folder:host',
                 'esxi_type:host',
                 'esxi_url:127.0.0.1:8989',
             ]
@@ -97,9 +94,6 @@ def test_vcsim_external_host_tags(vcsim_instance, datadog_agent, dd_run_check):
         'ha-host_VM0',
         {
             'esxi': [
-                'esxi_datacenter:ha-datacenter',
-                'esxi_folder:ha-folder-root',
-                'esxi_folder:vm',
                 'esxi_type:vm',
                 'esxi_host:localhost.localdomain',
                 'esxi_url:127.0.0.1:8989',
@@ -110,9 +104,6 @@ def test_vcsim_external_host_tags(vcsim_instance, datadog_agent, dd_run_check):
         'ha-host_VM1',
         {
             'esxi': [
-                'esxi_datacenter:ha-datacenter',
-                'esxi_folder:ha-folder-root',
-                'esxi_folder:vm',
                 'esxi_type:vm',
                 'esxi_host:localhost.localdomain',
                 'esxi_url:127.0.0.1:8989',
@@ -129,16 +120,10 @@ def test_esxi_resource_count_metrics(vcsim_instance, dd_run_check, aggregator):
 
     host_tags = [
         'esxi_compute:localhost.localdomain',
-        'esxi_datacenter:ha-datacenter',
-        'esxi_folder:ha-folder-root',
-        'esxi_folder:host',
         'esxi_type:host',
         'esxi_url:127.0.0.1:8989',
     ]
     vm_tags = [
-        'esxi_datacenter:ha-datacenter',
-        'esxi_folder:ha-folder-root',
-        'esxi_folder:vm',
         'esxi_type:vm',
         'esxi_host:localhost.localdomain',
         'esxi_url:127.0.0.1:8989',