From 65dcaef668ecd005e44e4d82039da2615d1d5237 Mon Sep 17 00:00:00 2001 From: steveny91 Date: Tue, 4 Jun 2024 09:24:12 -0400 Subject: [PATCH 1/8] test bump confluent_kafka --- .builders/images/linux-aarch64/build_script.sh | 2 +- .builders/images/linux-x86_64/build_script.sh | 2 +- .builders/images/macos-x86_64/extra_build.sh | 2 +- agent_requirements.in | 2 +- kafka_consumer/hatch.toml | 2 +- kafka_consumer/pyproject.toml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.builders/images/linux-aarch64/build_script.sh b/.builders/images/linux-aarch64/build_script.sh index 5f25c2b3596a5..0675c46311dce 100644 --- a/.builders/images/linux-aarch64/build_script.sh +++ b/.builders/images/linux-aarch64/build_script.sh @@ -21,7 +21,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then LDFLAGS="${LDFLAGS} -L/usr/local/lib -lkrb5 -lgssapi_krb5 -llmdb" \ DOWNLOAD_URL="https://github.com/confluentinc/librdkafka/archive/refs/tags/v{{version}}.tar.gz" \ VERSION="${kafka_version}" \ - SHA256="2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12" \ + SHA256="d645e47d961db47f1ead29652606a502bdd2a880c85c1e060e94eea040f1a19a" \ RELATIVE_PATH="librdkafka-{{version}}" \ bash install-from-source.sh --enable-sasl --enable-curl always_build+=("confluent-kafka") diff --git a/.builders/images/linux-x86_64/build_script.sh b/.builders/images/linux-x86_64/build_script.sh index ca23f2eac13c3..25cbcd668b0ee 100644 --- a/.builders/images/linux-x86_64/build_script.sh +++ b/.builders/images/linux-x86_64/build_script.sh @@ -18,7 +18,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then LDFLAGS="${LDFLAGS} -L/usr/local/lib -lkrb5 -lgssapi_krb5 -llmdb" \ DOWNLOAD_URL="https://github.com/confluentinc/librdkafka/archive/refs/tags/v{{version}}.tar.gz" \ VERSION="${kafka_version}" \ - SHA256="2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12" \ + SHA256="d645e47d961db47f1ead29652606a502bdd2a880c85c1e060e94eea040f1a19a" \ RELATIVE_PATH="librdkafka-{{version}}" \ bash install-from-source.sh --enable-sasl --enable-curl always_build+=("confluent-kafka") diff --git a/.builders/images/macos-x86_64/extra_build.sh b/.builders/images/macos-x86_64/extra_build.sh index f2fcff48b96a0..55deb0ef2e797 100644 --- a/.builders/images/macos-x86_64/extra_build.sh +++ b/.builders/images/macos-x86_64/extra_build.sh @@ -13,7 +13,7 @@ if [[ "${DD_BUILD_PYTHON_VERSION}" == "3" ]]; then LDFLAGS="${LDFLAGS} -L${DD_PREFIX_PATH}/lib -lgssapi_krb5 -llmdb" \ DOWNLOAD_URL="https://github.com/confluentinc/librdkafka/archive/refs/tags/v{{version}}.tar.gz" \ VERSION="${kafka_version}" \ - SHA256="2d49c35c77eeb3d42fa61c43757fcbb6a206daa560247154e60642bcdcc14d12" \ + SHA256="d645e47d961db47f1ead29652606a502bdd2a880c85c1e060e94eea040f1a19a" \ RELATIVE_PATH="librdkafka-{{version}}" \ bash install-from-source.sh --prefix="${DD_PREFIX_PATH}" --enable-sasl --enable-curl diff --git a/agent_requirements.in b/agent_requirements.in index 4ae43e45aa22a..524555a30ffbf 100644 --- a/agent_requirements.in +++ b/agent_requirements.in @@ -17,7 +17,7 @@ clickhouse-cityhash==1.0.2.4; python_version > '3.0' clickhouse-driver==0.2.0; python_version < '3.0' clickhouse-driver==0.2.7; python_version > '3.0' cm-client==45.0.4 -confluent-kafka==2.3.0; python_version > '3.0' +confluent-kafka==2.4.0; python_version > '3.0' contextlib2==0.6.0.post1; python_version < '3.0' cryptography==3.3.2; python_version < '3.0' cryptography==42.0.6; python_version > '3.0' diff --git a/kafka_consumer/hatch.toml b/kafka_consumer/hatch.toml index 557c363b02d56..05822b8cb45e1 100644 --- a/kafka_consumer/hatch.toml +++ b/kafka_consumer/hatch.toml @@ -5,7 +5,7 @@ # Also bump the LIBRDKAFKA_VERSION version in this file post-install-commands = [ "python -m pip uninstall -y confluent-kafka", - "python -m pip install --no-binary confluent-kafka confluent-kafka==2.3.0", + "python -m pip install --no-binary confluent-kafka confluent-kafka==2.4.0", ] [envs.default.env-vars] diff --git a/kafka_consumer/pyproject.toml b/kafka_consumer/pyproject.toml index 08185ae10a00b..eefa4a186923f 100644 --- a/kafka_consumer/pyproject.toml +++ b/kafka_consumer/pyproject.toml @@ -38,7 +38,7 @@ license = "BSD-3-Clause" deps = [ # confluent-kafka is built in omnibus, so bumping it here will have no real effect # if you bump this version, also bump the one in the `hatch.toml` file - "confluent-kafka==2.3.0; python_version > '3.0'", + "confluent-kafka==2.4.0; python_version > '3.0'", ] [project.urls] From cba56e828087d0863f179f8dfc275face94d908d Mon Sep 17 00:00:00 2001 From: steveny91 Date: Tue, 4 Jun 2024 09:37:55 -0400 Subject: [PATCH 2/8] bump build script --- .ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh b/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh index bfa229f984597..b082c6625838d 100755 --- a/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh +++ b/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh @@ -8,7 +8,7 @@ sudo apt install -y --no-install-recommends build-essential libkrb5-dev wget sof # Install librdkafka from source since no binaries are available for the distribution we use on the CI: git clone https://github.com/confluentinc/librdkafka cd librdkafka -git checkout v2.3.0 +git checkout v2.4.0 sudo ./configure --install-deps --prefix=/usr make sudo make install From 07982f423555e340fa9e8163ba124471fccb5312 Mon Sep 17 00:00:00 2001 From: steveny91 Date: Tue, 4 Jun 2024 09:40:06 -0400 Subject: [PATCH 3/8] add changelog --- kafka_consumer/changelog.d/17726.added | 1 + 1 file changed, 1 insertion(+) create mode 100644 kafka_consumer/changelog.d/17726.added diff --git a/kafka_consumer/changelog.d/17726.added b/kafka_consumer/changelog.d/17726.added new file mode 100644 index 0000000000000..ed492c1f0669e --- /dev/null +++ b/kafka_consumer/changelog.d/17726.added @@ -0,0 +1 @@ +Update confluent_kafka and librdkafka From 981568aca93179fc9579b64b531478470f452dbd Mon Sep 17 00:00:00 2001 From: steveny91 Date: Tue, 4 Jun 2024 11:53:51 -0400 Subject: [PATCH 4/8] test adding ssl --- .ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh b/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh index b082c6625838d..3d93bf70a85ac 100755 --- a/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh +++ b/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh @@ -3,13 +3,13 @@ set -ex sudo apt update -sudo apt install -y --no-install-recommends build-essential libkrb5-dev wget software-properties-common lsb-release gcc make python3 python3-pip python3-dev libsasl2-modules-gssapi-mit krb5-user +sudo apt install -y --no-install-recommends build-essential libkrb5-dev wget software-properties-common lsb-release gcc make python3 python3-pip python3-dev libsasl2-modules-gssapi-mit krb5-user libsasl2-dev # Install librdkafka from source since no binaries are available for the distribution we use on the CI: git clone https://github.com/confluentinc/librdkafka cd librdkafka git checkout v2.4.0 -sudo ./configure --install-deps --prefix=/usr +sudo ./configure --install-deps --prefix=/usr --enable-ssl make sudo make install From bbde51ac14ef817340ae3465b905f668b5ba3a53 Mon Sep 17 00:00:00 2001 From: steveny91 Date: Tue, 4 Jun 2024 14:06:18 -0400 Subject: [PATCH 5/8] enable sasl --- .ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh b/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh index 3d93bf70a85ac..2e7278d6b2be6 100755 --- a/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh +++ b/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh @@ -9,7 +9,7 @@ sudo apt install -y --no-install-recommends build-essential libkrb5-dev wget sof git clone https://github.com/confluentinc/librdkafka cd librdkafka git checkout v2.4.0 -sudo ./configure --install-deps --prefix=/usr --enable-ssl +sudo ./configure --install-deps --prefix=/usr --enable-ssl --enable-sasl make sudo make install From 7e9fe0684955f8f9252d8c36994409f2e50103b2 Mon Sep 17 00:00:00 2001 From: steveny91 Date: Tue, 4 Jun 2024 14:44:36 -0400 Subject: [PATCH 6/8] enable sasl --- .ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh b/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh index 2e7278d6b2be6..edca32b062056 100755 --- a/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh +++ b/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh @@ -3,13 +3,13 @@ set -ex sudo apt update -sudo apt install -y --no-install-recommends build-essential libkrb5-dev wget software-properties-common lsb-release gcc make python3 python3-pip python3-dev libsasl2-modules-gssapi-mit krb5-user libsasl2-dev +sudo apt install -y --no-install-recommends build-essential libkrb5-dev wget software-properties-common lsb-release gcc make python3 python3-pip python3-dev libsasl2-modules-gssapi-mit krb5-user # Install librdkafka from source since no binaries are available for the distribution we use on the CI: git clone https://github.com/confluentinc/librdkafka cd librdkafka git checkout v2.4.0 -sudo ./configure --install-deps --prefix=/usr --enable-ssl --enable-sasl +sudo ./configure --install-deps --prefix=/usr --enable-ssl --enable-sasl --enable-gssapi make sudo make install From ecc70b559797dfc9d680523e8dbe3787f1d91988 Mon Sep 17 00:00:00 2001 From: steveny91 Date: Tue, 4 Jun 2024 15:12:54 -0400 Subject: [PATCH 7/8] enable sasl --- .ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh b/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh index edca32b062056..bb2cbba955a95 100755 --- a/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh +++ b/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh @@ -3,7 +3,7 @@ set -ex sudo apt update -sudo apt install -y --no-install-recommends build-essential libkrb5-dev wget software-properties-common lsb-release gcc make python3 python3-pip python3-dev libsasl2-modules-gssapi-mit krb5-user +sudo apt install -y --no-install-recommends build-essential libkrb5-dev wget software-properties-common lsb-release gcc make python3 python3-pip python3-dev libsasl2-modules-gssapi-mit krb5-user libsasl2-dev # Install librdkafka from source since no binaries are available for the distribution we use on the CI: git clone https://github.com/confluentinc/librdkafka From 34dd50b017bf8a68fd02cbad744e78d1ba502d6e Mon Sep 17 00:00:00 2001 From: steveny91 Date: Wed, 5 Jun 2024 12:36:30 -0400 Subject: [PATCH 8/8] remove unneeded --- .ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh b/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh index bb2cbba955a95..b082c6625838d 100755 --- a/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh +++ b/.ddev/ci/scripts/kafka_consumer/linux/32_install_kerberos.sh @@ -3,13 +3,13 @@ set -ex sudo apt update -sudo apt install -y --no-install-recommends build-essential libkrb5-dev wget software-properties-common lsb-release gcc make python3 python3-pip python3-dev libsasl2-modules-gssapi-mit krb5-user libsasl2-dev +sudo apt install -y --no-install-recommends build-essential libkrb5-dev wget software-properties-common lsb-release gcc make python3 python3-pip python3-dev libsasl2-modules-gssapi-mit krb5-user # Install librdkafka from source since no binaries are available for the distribution we use on the CI: git clone https://github.com/confluentinc/librdkafka cd librdkafka git checkout v2.4.0 -sudo ./configure --install-deps --prefix=/usr --enable-ssl --enable-sasl --enable-gssapi +sudo ./configure --install-deps --prefix=/usr make sudo make install