From 0dd70aca62176667f4e2caaec2131c5a5c7a3d92 Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Mon, 26 May 2025 16:00:43 +0800 Subject: [PATCH 1/7] Use java21 for Pulsar 4.0+ in java runner image --- images/pulsar-functions-base-runner/Dockerfile | 5 ++++- images/pulsar-functions-java-runner/pulsarctl.Dockerfile | 7 +++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/images/pulsar-functions-base-runner/Dockerfile b/images/pulsar-functions-base-runner/Dockerfile index 622d3be21..36727114d 100644 --- a/images/pulsar-functions-base-runner/Dockerfile +++ b/images/pulsar-functions-base-runner/Dockerfile @@ -41,8 +41,11 @@ RUN echo "VERSION_TAG=${VERSION_TAG}" && \ elif [ $VERSION_MAJOR -eq 2 ] && [ $VERSION_MINOR -eq 11 ]; then \ echo "Pulsar version is 2.11, use java 17" && \ export JRE_PACKAGE_NAME='openjdk17 gcompat'; \ + elif [ $VERSION_MAJOR -eq 3 ]; then \ + echo "Pulsar version is 3.x, use java 17 instead" && \ + export JRE_PACKAGE_NAME='openjdk17 gcompat'; \ else \ - echo "Pulsar version is not in the list, use java 17 instead" && \ + echo "Pulsar version is not in the list, use java 21 instead" && \ export JRE_PACKAGE_NAME='openjdk17 gcompat'; \ fi && \ apk update && apk add --no-cache $JRE_PACKAGE_NAME bash diff --git a/images/pulsar-functions-java-runner/pulsarctl.Dockerfile b/images/pulsar-functions-java-runner/pulsarctl.Dockerfile index 2545190b4..c42314055 100644 --- a/images/pulsar-functions-java-runner/pulsarctl.Dockerfile +++ b/images/pulsar-functions-java-runner/pulsarctl.Dockerfile @@ -25,9 +25,12 @@ RUN echo "VERSION_TAG=${VERSION_TAG}" && \ elif [ $VERSION_MAJOR -eq 2 ] && [ $VERSION_MINOR -eq 11 ]; then \ echo "Pulsar version is 2.11, use java 17" && \ export JRE_PACKAGE_NAME='openjdk17 gcompat'; \ - else \ - echo "Pulsar version is not in the list, use java 17 instead" && \ + elif [ $VERSION_MAJOR -eq 3 ]; then \ + echo "Pulsar version is 3.x, use java 17" && \ export JRE_PACKAGE_NAME='openjdk17 gcompat'; \ + else \ + echo "Pulsar version is not in the list, use java 21 instead" && \ + export JRE_PACKAGE_NAME='openjdk21 gcompat'; \ fi && \ apk update && apk add --no-cache $JRE_PACKAGE_NAME From 4d02faadd3c7ad0448139f92c897b20d0be6e90b Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Mon, 26 May 2025 16:47:31 +0800 Subject: [PATCH 2/7] Fix kinesis --- .../pulsar-functions-base-runner/Dockerfile | 22 +++++++++++++++++++ .../pulsarctl.Dockerfile | 22 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/images/pulsar-functions-base-runner/Dockerfile b/images/pulsar-functions-base-runner/Dockerfile index 36727114d..492a37197 100644 --- a/images/pulsar-functions-base-runner/Dockerfile +++ b/images/pulsar-functions-base-runner/Dockerfile @@ -67,4 +67,26 @@ RUN rm -rf /pulsar/bin/pulsar-detector || true ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE ENV java.io.tmpdir=/pulsar/tmp/ +# Copy the kinesis_producer native executable compiled for Alpine musl to the pulsar-all image +# This is required to support the Pulsar IO Kinesis sink connector +COPY --from=pulsar /opt/amazon-kinesis-producer/bin/kinesis_producer /opt/amazon-kinesis-producer/bin/.os_info /opt/amazon-kinesis-producer/bin/.build_time /opt/amazon-kinesis-producer/bin/.revision /opt/amazon-kinesis-producer/bin/.system_info /opt/amazon-kinesis-producer/bin/.version /opt/amazon-kinesis-producer/bin/ +# Set the environment variable to point to the kinesis_producer native executable +ENV PULSAR_IO_KINESIS_KPL_PATH=/opt/amazon-kinesis-producer/bin/kinesis_producer +# Install the required dependencies for the kinesis_producer native executable +USER 0 +RUN apk update && apk add --no-cache \ + brotli-libs \ + c-ares \ + libcrypto3 \ + libcurl \ + libgcc \ + libidn2 \ + libpsl \ + libssl3 \ + libunistring \ + nghttp2-libs \ + zlib \ + zstd-libs \ + libuuid + WORKDIR /pulsar diff --git a/images/pulsar-functions-java-runner/pulsarctl.Dockerfile b/images/pulsar-functions-java-runner/pulsarctl.Dockerfile index c42314055..569ec11a1 100644 --- a/images/pulsar-functions-java-runner/pulsarctl.Dockerfile +++ b/images/pulsar-functions-java-runner/pulsarctl.Dockerfile @@ -56,4 +56,26 @@ ENV java.io.tmpdir=/pulsar/tmp/ WORKDIR /pulsar +# Copy the kinesis_producer native executable compiled for Alpine musl to the pulsar-all image +# This is required to support the Pulsar IO Kinesis sink connector +COPY --from=pulsar /opt/amazon-kinesis-producer/bin/kinesis_producer /opt/amazon-kinesis-producer/bin/.os_info /opt/amazon-kinesis-producer/bin/.build_time /opt/amazon-kinesis-producer/bin/.revision /opt/amazon-kinesis-producer/bin/.system_info /opt/amazon-kinesis-producer/bin/.version /opt/amazon-kinesis-producer/bin/ +# Set the environment variable to point to the kinesis_producer native executable +ENV PULSAR_IO_KINESIS_KPL_PATH=/opt/amazon-kinesis-producer/bin/kinesis_producer +# Install the required dependencies for the kinesis_producer native executable +USER 0 +RUN apk update && apk add --no-cache \ + brotli-libs \ + c-ares \ + libcrypto3 \ + libcurl \ + libgcc \ + libidn2 \ + libpsl \ + libssl3 \ + libunistring \ + nghttp2-libs \ + zlib \ + zstd-libs \ + libuuid + USER $USER From 5bebe1db2818f70a760dcf3cf5356619cba668a4 Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Mon, 26 May 2025 16:49:36 +0800 Subject: [PATCH 3/7] Update --- images/pulsar-functions-base-runner/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/pulsar-functions-base-runner/Dockerfile b/images/pulsar-functions-base-runner/Dockerfile index 492a37197..00218e853 100644 --- a/images/pulsar-functions-base-runner/Dockerfile +++ b/images/pulsar-functions-base-runner/Dockerfile @@ -46,7 +46,7 @@ RUN echo "VERSION_TAG=${VERSION_TAG}" && \ export JRE_PACKAGE_NAME='openjdk17 gcompat'; \ else \ echo "Pulsar version is not in the list, use java 21 instead" && \ - export JRE_PACKAGE_NAME='openjdk17 gcompat'; \ + export JRE_PACKAGE_NAME='openjdk21 gcompat'; \ fi && \ apk update && apk add --no-cache $JRE_PACKAGE_NAME bash From 0efbbde24d5d2c6fb41aec09db37658922542a4f Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Mon, 26 May 2025 17:08:18 +0800 Subject: [PATCH 4/7] Remove unused code --- images/pulsar-functions-base-runner/pulsarctl.Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/images/pulsar-functions-base-runner/pulsarctl.Dockerfile b/images/pulsar-functions-base-runner/pulsarctl.Dockerfile index 564a10cf3..8ee8ba37f 100644 --- a/images/pulsar-functions-base-runner/pulsarctl.Dockerfile +++ b/images/pulsar-functions-base-runner/pulsarctl.Dockerfile @@ -1,6 +1,3 @@ -ARG PULSAR_IMAGE -ARG PULSAR_IMAGE_TAG -FROM ${PULSAR_IMAGE}:${PULSAR_IMAGE_TAG} as pulsar FROM alpine:3.20 as functions-runner ENV GID=10001 From 852028cbb0c2d2ae56db5c813ac1a4ad79979ba8 Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Mon, 26 May 2025 17:33:08 +0800 Subject: [PATCH 5/7] Update --- images/pulsar-functions-base-runner/Dockerfile | 4 ++-- images/pulsar-functions-java-runner/pulsarctl.Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/images/pulsar-functions-base-runner/Dockerfile b/images/pulsar-functions-base-runner/Dockerfile index 00218e853..52b208bdd 100644 --- a/images/pulsar-functions-base-runner/Dockerfile +++ b/images/pulsar-functions-base-runner/Dockerfile @@ -67,9 +67,9 @@ RUN rm -rf /pulsar/bin/pulsar-detector || true ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE ENV java.io.tmpdir=/pulsar/tmp/ -# Copy the kinesis_producer native executable compiled for Alpine musl to the pulsar-all image +# Copy the kinesis_producer native executable compiled for Alpine musl to the runner image # This is required to support the Pulsar IO Kinesis sink connector -COPY --from=pulsar /opt/amazon-kinesis-producer/bin/kinesis_producer /opt/amazon-kinesis-producer/bin/.os_info /opt/amazon-kinesis-producer/bin/.build_time /opt/amazon-kinesis-producer/bin/.revision /opt/amazon-kinesis-producer/bin/.system_info /opt/amazon-kinesis-producer/bin/.version /opt/amazon-kinesis-producer/bin/ +COPY --from=pulsar --chown=$UID:$GID /opt/amazon-kinesis-produce* /opt/amazon-kinesis-producer # Set the environment variable to point to the kinesis_producer native executable ENV PULSAR_IO_KINESIS_KPL_PATH=/opt/amazon-kinesis-producer/bin/kinesis_producer # Install the required dependencies for the kinesis_producer native executable diff --git a/images/pulsar-functions-java-runner/pulsarctl.Dockerfile b/images/pulsar-functions-java-runner/pulsarctl.Dockerfile index 569ec11a1..beba98af4 100644 --- a/images/pulsar-functions-java-runner/pulsarctl.Dockerfile +++ b/images/pulsar-functions-java-runner/pulsarctl.Dockerfile @@ -58,7 +58,7 @@ WORKDIR /pulsar # Copy the kinesis_producer native executable compiled for Alpine musl to the pulsar-all image # This is required to support the Pulsar IO Kinesis sink connector -COPY --from=pulsar /opt/amazon-kinesis-producer/bin/kinesis_producer /opt/amazon-kinesis-producer/bin/.os_info /opt/amazon-kinesis-producer/bin/.build_time /opt/amazon-kinesis-producer/bin/.revision /opt/amazon-kinesis-producer/bin/.system_info /opt/amazon-kinesis-producer/bin/.version /opt/amazon-kinesis-producer/bin/ +COPY --from=pulsar --chown=$UID:$GID /opt/amazon-kinesis-produce* /opt/amazon-kinesis-producer # Set the environment variable to point to the kinesis_producer native executable ENV PULSAR_IO_KINESIS_KPL_PATH=/opt/amazon-kinesis-producer/bin/kinesis_producer # Install the required dependencies for the kinesis_producer native executable From 8b0b64286d62829f50096a10472cf0dc9cde9f5d Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Tue, 27 May 2025 15:57:43 +0800 Subject: [PATCH 6/7] Address comments --- images/pulsar-functions-base-runner/Dockerfile | 5 +++-- images/pulsar-functions-base-runner/pulsarctl.Dockerfile | 2 +- images/pulsar-functions-java-runner/pulsarctl.Dockerfile | 3 ++- operator.Dockerfile | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/images/pulsar-functions-base-runner/Dockerfile b/images/pulsar-functions-base-runner/Dockerfile index 52b208bdd..2ddc62355 100644 --- a/images/pulsar-functions-base-runner/Dockerfile +++ b/images/pulsar-functions-base-runner/Dockerfile @@ -1,7 +1,8 @@ ARG PULSAR_IMAGE ARG PULSAR_IMAGE_TAG FROM ${PULSAR_IMAGE}:${PULSAR_IMAGE_TAG} as pulsar -FROM alpine:3.20 as functions-runner +FROM apachepulsar/pulsar-io-kinesis-sink-kinesis_producer:0.15.12 as pulsar-io-kinesis-sink-kinesis_producer +FROM alpine:3.21 as functions-runner ENV GID=10001 ENV UID=10000 @@ -69,7 +70,7 @@ ENV java.io.tmpdir=/pulsar/tmp/ # Copy the kinesis_producer native executable compiled for Alpine musl to the runner image # This is required to support the Pulsar IO Kinesis sink connector -COPY --from=pulsar --chown=$UID:$GID /opt/amazon-kinesis-produce* /opt/amazon-kinesis-producer +COPY --from=pulsar-io-kinesis-sink-kinesis_producer --chown=$UID:$GID /opt/amazon-kinesis-producer/bin/kinesis_producer /opt/amazon-kinesis-producer/bin/.os_info /opt/amazon-kinesis-producer/bin/.build_time /opt/amazon-kinesis-producer/bin/.revision /opt/amazon-kinesis-producer/bin/.system_info /opt/amazon-kinesis-producer/bin/.version /opt/amazon-kinesis-producer/bin/ # Set the environment variable to point to the kinesis_producer native executable ENV PULSAR_IO_KINESIS_KPL_PATH=/opt/amazon-kinesis-producer/bin/kinesis_producer # Install the required dependencies for the kinesis_producer native executable diff --git a/images/pulsar-functions-base-runner/pulsarctl.Dockerfile b/images/pulsar-functions-base-runner/pulsarctl.Dockerfile index 8ee8ba37f..08b8911e9 100644 --- a/images/pulsar-functions-base-runner/pulsarctl.Dockerfile +++ b/images/pulsar-functions-base-runner/pulsarctl.Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.20 as functions-runner +FROM alpine:3.21 as functions-runner ENV GID=10001 ENV UID=10000 diff --git a/images/pulsar-functions-java-runner/pulsarctl.Dockerfile b/images/pulsar-functions-java-runner/pulsarctl.Dockerfile index beba98af4..801b081b6 100644 --- a/images/pulsar-functions-java-runner/pulsarctl.Dockerfile +++ b/images/pulsar-functions-java-runner/pulsarctl.Dockerfile @@ -1,6 +1,7 @@ ARG PULSAR_IMAGE ARG PULSAR_IMAGE_TAG FROM ${PULSAR_IMAGE}:${PULSAR_IMAGE_TAG} as pulsar +FROM apachepulsar/pulsar-io-kinesis-sink-kinesis_producer:0.15.12 as pulsar-io-kinesis-sink-kinesis_producer FROM pulsar-functions-pulsarctl-runner-base:latest ARG PULSAR_IMAGE_TAG @@ -58,7 +59,7 @@ WORKDIR /pulsar # Copy the kinesis_producer native executable compiled for Alpine musl to the pulsar-all image # This is required to support the Pulsar IO Kinesis sink connector -COPY --from=pulsar --chown=$UID:$GID /opt/amazon-kinesis-produce* /opt/amazon-kinesis-producer +COPY --from=pulsar-io-kinesis-sink-kinesis_producer --chown=$UID:$GID /opt/amazon-kinesis-producer/bin/kinesis_producer /opt/amazon-kinesis-producer/bin/.os_info /opt/amazon-kinesis-producer/bin/.build_time /opt/amazon-kinesis-producer/bin/.revision /opt/amazon-kinesis-producer/bin/.system_info /opt/amazon-kinesis-producer/bin/.version /opt/amazon-kinesis-producer/bin/ # Set the environment variable to point to the kinesis_producer native executable ENV PULSAR_IO_KINESIS_KPL_PATH=/opt/amazon-kinesis-producer/bin/kinesis_producer # Install the required dependencies for the kinesis_producer native executable diff --git a/operator.Dockerfile b/operator.Dockerfile index 7624968d2..9c8587fca 100644 --- a/operator.Dockerfile +++ b/operator.Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.20 +FROM alpine:3.21 ENV GID=10001 ENV UID=10000 From 9ba6fd72b387da52ad210030a884be75d29f76bb Mon Sep 17 00:00:00 2001 From: jiangpengcheng Date: Wed, 28 May 2025 09:18:57 +0800 Subject: [PATCH 7/7] Update images/pulsar-functions-java-runner/pulsarctl.Dockerfile Co-authored-by: Lari Hotari --- images/pulsar-functions-java-runner/pulsarctl.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/pulsar-functions-java-runner/pulsarctl.Dockerfile b/images/pulsar-functions-java-runner/pulsarctl.Dockerfile index 801b081b6..78d07a472 100644 --- a/images/pulsar-functions-java-runner/pulsarctl.Dockerfile +++ b/images/pulsar-functions-java-runner/pulsarctl.Dockerfile @@ -57,7 +57,7 @@ ENV java.io.tmpdir=/pulsar/tmp/ WORKDIR /pulsar -# Copy the kinesis_producer native executable compiled for Alpine musl to the pulsar-all image +# Copy the kinesis_producer native executable compiled for Alpine musl to the image # This is required to support the Pulsar IO Kinesis sink connector COPY --from=pulsar-io-kinesis-sink-kinesis_producer --chown=$UID:$GID /opt/amazon-kinesis-producer/bin/kinesis_producer /opt/amazon-kinesis-producer/bin/.os_info /opt/amazon-kinesis-producer/bin/.build_time /opt/amazon-kinesis-producer/bin/.revision /opt/amazon-kinesis-producer/bin/.system_info /opt/amazon-kinesis-producer/bin/.version /opt/amazon-kinesis-producer/bin/ # Set the environment variable to point to the kinesis_producer native executable