Merge branch 'master' into aerospike-agent-and-dashboards

Author: mphanias

.builders/images/linux-aarch64/Dockerfile

@@ -36,11 +36,11 @@ RUN yum install -y perl-IPC-Cmd perl-CPANPLUS && \
  ldconfig
 
 # Compile and install Python 3
-ENV PYTHON3_VERSION=3.12.9
+ENV PYTHON3_VERSION=3.12.10
 RUN yum install -y libffi-devel && \
  DOWNLOAD_URL="https://python.org/ftp/python/{{version}}/Python-{{version}}.tgz" \
  VERSION="${PYTHON3_VERSION}" \
- SHA256="45313e4c5f0e8acdec9580161d565cf5fea578e3eabf25df7cc6355bf4afa1ee" \
+ SHA256="15d9c623abfd2165fe816ea1fb385d6ed8cf3c664661ab357f1782e3036a6dac" \
  RELATIVE_PATH="Python-{{version}}" \
  bash install-from-source.sh \
  --prefix=/opt/python/${PYTHON3_VERSION} \

.builders/images/linux-x86_64/Dockerfile

@@ -35,11 +35,11 @@ RUN yum install -y perl-IPC-Cmd perl-CPANPLUS && \
  ldconfig
 
 # Compile and install Python 3
-ENV PYTHON3_VERSION=3.12.9
+ENV PYTHON3_VERSION=3.12.10
 RUN yum install -y libffi-devel && \
  DOWNLOAD_URL="https://python.org/ftp/python/{{version}}/Python-{{version}}.tgz" \
  VERSION="${PYTHON3_VERSION}" \
- SHA256="45313e4c5f0e8acdec9580161d565cf5fea578e3eabf25df7cc6355bf4afa1ee" \
+ SHA256="15d9c623abfd2165fe816ea1fb385d6ed8cf3c664661ab357f1782e3036a6dac" \
  RELATIVE_PATH="Python-{{version}}" \
  bash install-from-source.sh --prefix=/opt/python/${PYTHON3_VERSION} --with-ensurepip=yes --enable-ipv6 --with-dbmliborder=
 ENV PATH="/opt/python/${PYTHON3_VERSION}/bin:${PATH}"

.builders/images/windows-x86_64/Dockerfile

@@ -82,11 +82,11 @@ RUN Get-RemoteFile `
     Approve-File -Path $($Env:USERPROFILE + '\.cargo\bin\rustc.exe') -Hash $Env:RUSTC_HASH
 
 # Install Python 3
-ENV PYTHON_VERSION="3.12.9"
+ENV PYTHON_VERSION="3.12.10"
 RUN Get-RemoteFile `
       -Uri https://www.python.org/ftp/python/$Env:PYTHON_VERSION/python-$Env:PYTHON_VERSION-amd64.exe `
       -Path python-$Env:PYTHON_VERSION-amd64.exe `
-      -Hash '2a52993092a19cfdffe126e2eeac46a4265e25705614546604ad44988e040c0f'; `
+      -Hash '67b5635e80ea51072b87941312d00ec8927c4db9ba18938f7ad2d27b328b95fb'; `
     Start-Process -Wait python-$Env:PYTHON_VERSION-amd64.exe -ArgumentList '/quiet', 'InstallAllUsers=1'; `
     Remove-Item python-$Env:PYTHON_VERSION-amd64.exe; `
     & 'C:\Program Files\Python312\python.exe' -m pip install --no-warn-script-location --upgrade pip; `

.github/CODEOWNERS

@@ -528,6 +528,11 @@ plaid/assets/logs/                                                  @DataDog/saa
 /openvpn/manifest.json                                              @DataDog/agent-integrations @DataDog/documentation
 /openvpn/assets/logs/                                               @DataDog/agent-integrations @DataDog/documentation @DataDog/logs-backend @DataDog/logs-core
 
+/watchguard_firebox/                                                  @DataDog/agent-integrations
+/watchguard_firebox/*.md                                              @DataDog/agent-integrations @DataDog/documentation
+/watchguard_firebox/manifest.json                                     @DataDog/agent-integrations @DataDog/documentation
+/watchguard_firebox/assets/logs/                                      @DataDog/agent-integrations @DataDog/documentation @DataDog/logs-backend @DataDog/logs-core
+
 /kaspersky/                                                          @DataDog/agent-integrations
 /kaspersky/*.md                                                      @DataDog/agent-integrations @DataDog/documentation
 /kaspersky/manifest.json                                             @DataDog/agent-integrations @DataDog/documentation

.github/workflows/config/labeler.yml

@@ -673,6 +673,8 @@ integration/vonage:
 - vonage/**/*
 integration/vsphere:
 - vsphere/**/*
+integration/watchguard_firebox:
+- watchguard_firebox/**/*
 integration/wazuh:
 - wazuh/**/*
 integration/weaviate:

.github/workflows/resolve-build-deps.yaml

@@ -194,7 +194,7 @@ jobs:
     - name: Set up Python
       env:
         # Despite the name, this is built for the macOS 11 SDK on arm64 and 10.9+ on intel
-        PYTHON3_DOWNLOAD_URL: "https://www.python.org/ftp/python/3.12.9/python-3.12.9-macos11.pkg"
+        PYTHON3_DOWNLOAD_URL: "https://www.python.org/ftp/python/3.12.10/python-3.12.10-macos11.pkg"
       run: |-
         curl "$PYTHON3_DOWNLOAD_URL" -o python3.pkg
         sudo installer -pkg python3.pkg -target /

mysql/changelog.d/20336.fixed

@@ -0,0 +1 @@
+Set lock_wait_timeout session variable lower in order to avoid stalling on acquiring metadata locks when explaining queries

mysql/datadog_checks/mysql/activity.py

@@ -18,7 +18,7 @@
 from datadog_checks.base.utils.tracking import tracked_method
 from datadog_checks.mysql.cursor import CommenterDictCursor
 
-from .util import DatabaseConfigurationError, connect_with_autocommit, get_truncation_state, warning_with_tags
+from .util import DatabaseConfigurationError, connect_with_session_variables, get_truncation_state, warning_with_tags
 
 try:
     import datadog_agent
@@ -387,7 +387,7 @@ def _get_db_connection(self):
         pymysql connections are not thread safe, so we can't reuse the same connection from the main check.
         """
         if not self._db:
-            self._db = connect_with_autocommit(**self._connection_args)
+            self._db = connect_with_session_variables(**self._connection_args)
         return self._db
 
     def _close_db_conn(self):

mysql/datadog_checks/mysql/metadata.py

@@ -10,7 +10,7 @@
 from datadog_checks.mysql.cursor import CommenterDictCursor
 from datadog_checks.mysql.databases_data import DEFAULT_DATABASES_DATA_COLLECTION_INTERVAL, DatabasesData
 
-from .util import connect_with_autocommit
+from .util import connect_with_session_variables
 
 try:
     import datadog_agent
@@ -94,7 +94,7 @@ def get_db_connection(self):
         :return:
         """
         if not self._db:
-            self._db = connect_with_autocommit(**self._connection_args)
+            self._db = connect_with_session_variables(**self._connection_args)
         else:
             # Metadata checks runs far less frequently than other checks, and there are reports
             # that unused pymysql connections sometimes end up being closed unexpectedly.

mysql/datadog_checks/mysql/mysql.py

@@ -82,7 +82,7 @@
 )
 from .statement_samples import MySQLStatementSamples
 from .statements import MySQLStatementMetrics
-from .util import DatabaseConfigurationError, connect_with_autocommit  # noqa: F401
+from .util import DatabaseConfigurationError, connect_with_session_variables  # noqa: F401
 from .version_utils import get_version
 
 try:
@@ -535,7 +535,7 @@ def _connect(self):
         db = None
         try:
             connect_args = self._get_connection_args()
-            db = connect_with_autocommit(**connect_args)
+            db = connect_with_session_variables(**connect_args)
             self.log.debug("Connected to MySQL")
             self.service_check_tags = list(set(service_check_tags))
             self.service_check(

mysql/datadog_checks/mysql/statement_samples.py

@@ -34,7 +34,7 @@
 from .util import (
     DatabaseConfigurationError,
     StatementTruncationState,
-    connect_with_autocommit,
+    connect_with_session_variables,
     get_truncation_state,
     warning_with_tags,
 )
@@ -280,7 +280,7 @@ def _get_db_connection(self):
         :return:
         """
         if not self._db:
-            self._db = connect_with_autocommit(**self._connection_args)
+            self._db = connect_with_session_variables(**self._connection_args)
         return self._db
 
     def _close_db_conn(self):

mysql/datadog_checks/mysql/statements.py

@@ -20,7 +20,7 @@
 from datadog_checks.base.utils.tracking import tracked_method
 from datadog_checks.mysql.cursor import CommenterDictCursor
 
-from .util import DatabaseConfigurationError, connect_with_autocommit, warning_with_tags
+from .util import DatabaseConfigurationError, connect_with_session_variables, warning_with_tags
 
 try:
     import datadog_agent
@@ -105,7 +105,7 @@ def _get_db_connection(self):
         :return:
         """
         if not self._db:
-            self._db = connect_with_autocommit(**self._connection_args)
+            self._db = connect_with_session_variables(**self._connection_args)
         return self._db
 
     def _close_db_conn(self):

mysql/datadog_checks/mysql/util.py

@@ -47,15 +47,17 @@ def get_truncation_state(statement):
     return StatementTruncationState.truncated if truncated else StatementTruncationState.not_truncated
 
 
-def connect_with_autocommit(**connect_args):
+def connect_with_session_variables(**connect_args):
     db = pymysql.connect(**connect_args)
     with closing(db.cursor(CommenterCursor)) as cursor:
         # PyMYSQL only sets autocommit if it receives a different value from the server
         # see https://github.com/PyMySQL/PyMySQL/blob/bbd049f40db9c696574ce6f31669880042c56d79/pymysql/connections.py#L443-L447
         # but there are cases where the server will not send a correct value for autocommit, so we
         # set it explicitly to ensure it's set correctly
         cursor.execute("SET AUTOCOMMIT=1")
-
+        # Lower the lock wait timeout to avoid deadlocks on metadata locks. By default this is a year.
+        # https://dev.mysql.com/doc/refman/8.4/en/server-system-variables.html#sysvar_lock_wait_timeout
+        cursor.execute("SET LOCK_WAIT_TIMEOUT=5")
     return db
 
 

watchguard_firebox/CHANGELOG.md

@@ -0,0 +1,4 @@
+# CHANGELOG - WatchGuard Firebox
+
+<!-- towncrier release notes start -->
+

watchguard_firebox/README.md

@@ -0,0 +1,140 @@
+## Overview
+
+[WatchGuard Firebox][4] is a network security platform offering comprehensive threat protection, including firewall, networking, and other security features, suitable for various network environments, from small offices to large enterprises.
+
+This integration enriches and ingests the following events:
+
+- **Firewall Events**: Represents log messages generated by your Firebox for events that occur on the Firebox and for traffic managed by packet filter policies. In addition to normal traffic, this can include messages related to feature keys, subscription services, server load balancing, and other features configured on your Firebox.
+- **Networking Events**: Represents log messages generated for traffic related to the connections through your Firebox. This can include events related to interface activity, dynamic routing, PPPoE connections, and DHCP server requests.
+- **Management Events**: Represents log messages generated for activity on your Firebox. This includes when changes are made to the device configuration and Device Management user accounts, for user authentication to the Firebox, and actions related to system settings.
+- **VPN Events**: Represents log messages generated for processes related to the all VPNs configured on your Firebox. This includes changes to the VPN configuration, tunnel status, and daemon activity.
+- **Proxy Policy Events**: Represents log messages generated for traffic managed by the proxy policies configured on your Firebox. This can include events related to traffic through the proxy, proxy actions, and security services.
+
+This integration seamlessly collects all the above listed logs, channeling them into Datadog for analysis. Leveraging the built-in logs pipeline, these logs are parsed and enriched, enabling effortless search and analysis. The integration provides insight into firewall, networking, management, VPN, and proxy policy events through the out-of-the-box dashboards. Additionally, it includes ready-to-use Cloud SIEM detection rules for enhanced monitoring and security.
+
+## Setup
+
+### Installation
+
+To install the WatchGuard Firebox integration, run the following Agent installation command and the steps below. For more information, see the [Integration Management][5] documentation.
+
+**Note**: This step is not necessary for Agent version >= 7.66.0.
+
+Linux command:
+
+  ```shell
+  sudo -u dd-agent -- datadog-agent integration install datadog-watchguard-firebox==1.0.0
+  ```
+
+### Configuration
+
+#### Log collection
+
+1. Collecting logs is disabled by default in the Datadog Agent. Enable it in `datadog.yaml`:
+
+    ```yaml
+    logs_enabled: true
+    ```
+
+2. Add this configuration block to your `watchguard-firebox.d/conf.yaml` file to start collecting your WatchGuard Firebox logs. See the sample [watchguard-firebox.d/conf.yaml][6] for available configuration options.
+
+      ```yaml
+      logs:
+       - type: udp
+         port: <PORT>
+         service: watchguard-firebox
+         source: watchguard-firebox
+      ```
+
+      Note:
+      - PORT: Port should be identical to the port provided in **Configure Syslog Message Forwarding from WatchGuard Firebox Server**.
+      - It is recommended not to change the service and source values, as these parameters are integral to the pipeline's operation.
+
+3. [Restart the Agent][1].
+
+#### Configure Syslog Message Forwarding from WatchGuard Firebox Server
+
+1. Log in to **Fireware Web UI** with an administrator account.
+2. Select **System** > **Logging**.
+3. Select the **Syslog Server** tab.
+4. Select the **Send log messages to these syslog servers** check box.
+5. Click **Add** to open the Syslog Server dialog box.
+6. In the **IP Address** text box, enter the IP address of the server on which datadog-agent is installed.
+7. In the **Port** text box, enter the port you configured to receive the Firebox logs.
+8. From the **Log Format** drop-down list, select **Syslog**.
+9. Select **The time stamp** and **The serial number of the device** check boxes.
+10. Click **OK**.
+
+### Validation
+
+[Run the Agent's status subcommand][2] and look for `watchguard-firebox` under the Checks section.
+
+## Data Collected
+
+### Logs
+
+The WatchGuard Firebox integration collects firewall, networking, management, VPN, and proxy policy Events.
+
+### Metrics
+
+The WatchGuard Firebox integration does not include any metrics.
+
+### Events
+
+The WatchGuard Firebox integration does not include any events.
+
+## Troubleshooting
+
+### WatchGuard Firebox
+
+**Permission denied while port binding:**
+
+If you see a **Permission denied** error while port binding in the Agent logs, see the following instructions:
+
+   1. Binding to a port number under 1024 requires elevated permissions. Grant access to the port using the `setcap` command:
+
+      - Grant access to the port using the `setcap` command:
+
+         ```shell
+         sudo setcap CAP_NET_BIND_SERVICE=+ep /opt/datadog-agent/bin/agent/agent
+         ```
+
+      - Verify the setup is correct by running the `getcap` command:
+
+         ```shell
+         sudo getcap /opt/datadog-agent/bin/agent/agent
+         ```
+
+         With the expected output:
+
+         ```shell
+         /opt/datadog-agent/bin/agent/agent = cap_net_bind_service+ep
+         ```
+
+         **Note**: Re-run this `setcap` command every time you upgrade the Agent.
+
+   2. [Restart the Agent][1].
+
+**Data is not being collected:**
+
+Make sure that traffic is bypassed from the configured port if the firewall is enabled.
+
+**Port already in use:**
+
+If you see the **Port <PORT_NUMBER> Already in Use** error, see the following instructions. The example below is for a PORT_NUMBER equal to 514:
+
+On systems using Syslog, if the Agent listens for events on port 514, the following error can appear in the Agent logs: `Can't start UDP forwarder on port 514: listen udp :514: bind: address already in use`.
+
+This error occurs because by default, Syslog listens on port 514. To resolve this error, take **one** of the following steps:
+
+- Disable Syslog.
+- Configure the Agent to listen on a different, available port.
+
+For any further assistance, contact [Datadog support][3].
+
+[1]: https://docs.datadoghq.com/agent/guide/agent-commands/#start-stop-and-restart-the-agent
+[2]: https://docs.datadoghq.com/agent/guide/agent-commands/#agent-status-and-information
+[3]: https://docs.datadoghq.com/help/
+[4]: https://www.watchguard.com/wgrd-products/network-security
+[5]: https://docs.datadoghq.com/agent/guide/integration-management/?tab=linux#install
+[6]: https://github.com/DataDog/integrations-core/blob/master/watchguard-firebox/datadog_checks/watchguard-firebox/data/conf.yaml.example

watchguard_firebox/assets/configuration/spec.yaml

@@ -0,0 +1,10 @@
+name: WatchGuard Firebox
+files:
+- name: watchguard_firebox.yaml
+  options:
+  - template: logs
+    example:
+    - type: udp
+      port: <PORT>
+      service: watchguard-firebox
+      source: watchguard-firebox