[HWORKS-1722] Fix Jenkinsfile to also build webhook and controller images. Make it possible for image to be run by any user.

Jenkinsfile

@@ -5,6 +5,9 @@ pipeline {
     environment {
         VERSION = readFile "${env.WORKSPACE}/version.txt"
         BUILD_BRANCH = readFile "${env.WORKSPACE}/build_branch.txt"
+        DOCKER_REGISTRY = "docker.hops.works"
+        CONTROLLER_IMAGE = "${DOCKER_REGISTRY}/hopsworks/rss-controller:${VERSION}"
+        WEBHOOK_IMAGE = "${DOCKER_REGISTRY}/hopsworks/rss-webhook:${VERSION}"
     }
     stages {
         stage("checkout") {
@@ -26,18 +29,26 @@ pipeline {
                 sh """
                     set -ex
                     echo "Building RSS version ${VERSION} on branch ${BUILD_BRANCH}"
-                    docker login -u ${NEXUS_CREDS_USR} -p ${NEXUS_CREDS_PSW} docker.hops.works
+                    docker login -u ${NEXUS_CREDS_USR} -p ${NEXUS_CREDS_PSW} $DOCKER_REGISTRY
 
                     ./build_distribution.sh --spark3-profile spark3 --hadoop-profile hadoop3.2 --without-dashboard
                     cd deploy/kubernetes/docker ||  exit
-                    ./build.sh --hadoop-version 3.2.0.14-EE-RC0 --registry docker.hops.works --nexus-user $NEXUS_CREDS_USR --nexus-password $NEXUS_CREDS_PSW
+                    ./build.sh --hadoop-version 3.2.0.15-EE-SNAPSHOT --registry $DOCKER_REGISTRY --nexus-user $NEXUS_CREDS_USR --nexus-password $NEXUS_CREDS_PSW
                     cd ../../..
 
                     mkdir -p /opt/repository/master/rss/${VERSION}/
                     cp  client-spark/spark3-shaded/target/rss-client-spark3-shaded-${VERSION}.jar /opt/repository/master/rss/${VERSION}/
+
+                    # build the controller and webhook images
+                    cd deploy/kubernetes/operator ||  exit 1
+                    docker build . --progress=plain -t $CONTROLLER_IMAGE --build-arg MODULE=controller -f hack/Dockerfile
+                    docker build . --progress=plain -t $WEBHOOK_IMAGE --build-arg MODULE=webhook -f hack/Dockerfile
+                    # push the controller and webhook images
+                    docker push $CONTROLLER_IMAGE
+                    docker push $WEBHOOK_IMAGE
                 """
              }
             }
         }
     }
-}
+}

deploy/kubernetes/docker/Dockerfile

@@ -18,6 +18,7 @@ LABEL git-commit=${GIT_COMMIT}
 RUN useradd -ms /bin/bash rssadmin
 RUN mkdir -p /data/rssadmin/
 RUN chown -R rssadmin:rssadmin /data
+RUN chmod -R 777 /data
 USER rssadmin
 
 COPY rss-${RSS_VERSION}-hadoop${HADOOP_SHORT_VERSION}.tgz /data/rssadmin
@@ -26,9 +27,12 @@ RUN mv /data/rssadmin/rss-${RSS_VERSION}-hadoop${HADOOP_SHORT_VERSION} /data/rss
 RUN rm /data/rssadmin/rss/bin/rss-env.sh
 RUN rm -rf /data/rssadmin/rss-${RSS_VERSION}-hadoop${HADOOP_SHORT_VERSION}.tgz
 
-COPY rss-env.sh /data/rssadmin/rss/bin
 
-COPY start.sh /data/rssadmin/rss/bin
+RUN chmod -R 777 /data/rssadmin/rss
+
+COPY --chmod=755 rss-env.sh /data/rssadmin/rss/bin
+
+COPY --chmod=755 start.sh /data/rssadmin/rss/bin
 
 COPY $HADOOP_FILE /data/rssadmin
 RUN mkdir -p /data/rssadmin/hops-${HADOOP_VERSION}
@@ -38,7 +42,6 @@ RUN rm -rf /data/rssadmin/$HADOOP_FILE
 COPY hadoopconfig/ /data/rssadmin/hadoop/etc/hadoop
 
 ENV RSS_VERSION ${RSS_VERSION}
-ENV HADOOP_VERSION ${HADOOP_VERSION}
 
 WORKDIR /data/rssadmin
 CMD ["bash", "/data/rssadmin/rss/bin/start.sh"]

deploy/kubernetes/docker/build.sh

@@ -36,7 +36,7 @@ function exit_with_usage() {
 }
 
 REGISTRY="docker.io/library"
-HADOOP_VERSION=3.2.0.13-SNAPSHOT
+HADOOP_VERSION=3.2.0.15-EE-SNAPSHOT
 AUTHOR=$(whoami)
 # If you are based in China, you could pass --apache-mirror <a_mirror_url> when building this.
 APACHE_MIRROR="https://dlcdn.apache.org"
@@ -140,7 +140,7 @@ GIT_COMMIT=$(git describe --dirty --always --tags | sed 's/-/./g')
 IMAGE_VERSION=$(cat $RSS_DIR/version.txt | awk '{$1=$1; print}' | sed '/^$/d')
 IMAGE=$REGISTRY/rss-server:$IMAGE_VERSION
 echo "image version: ${IMAGE_VERSION}"
-IMAGE=$REGISTRY/rss:$IMAGE_VERSION
+IMAGE=$REGISTRY/hopsworks/rss:$IMAGE_VERSION
 echo "building image: $IMAGE"
 
 DOCKER_BUILDKIT=1 docker build --network=host -t "$IMAGE" \

deploy/kubernetes/operator/api/uniffle/v1alpha1/remoteshuffleservice_types.go

@@ -142,6 +142,29 @@ type ShuffleServerConfig struct {
 
 	// UpgradeStrategy defines upgrade strategy of shuffle servers.
 	UpgradeStrategy *ShuffleServerUpgradeStrategy `json:"upgradeStrategy"`
+
+	// volumeClaimTemplates is a list of claims that pods are allowed to reference.
+	// The StatefulSet controller is responsible for mapping network identities to
+	// claims in a way that maintains the identity of a pod. Every claim in
+	// this list must have at least one matching (by name) volumeMount in one
+	// container in the template. A claim in this list takes precedence over
+	// any volumes in the template, with the same name.
+	// +optional
+	VolumeClaimTemplates []ShuffleServerPersistentVolumeClaimTemplate `json:"volumeClaimTemplates,omitempty" protobuf:"bytes,4,rep,name=volumeClaimTemplates"`
+}
+
+type ShuffleServerPersistentVolumeClaimTemplate struct {
+	// May contain labels and annotations that will be copied into the PVC
+	// when creating it. No other fields are allowed and will be rejected during
+	// validation.
+	//
+	VolumeNameTemplate *string `json:"volumeNameTemplate"`
+
+	// The specification for the PersistentVolumeClaim. The entire content is
+	// copied unchanged into the PVC that gets created from this
+	// template. The same fields as in a PersistentVolumeClaim
+	// are also valid here.
+	Spec corev1.PersistentVolumeClaimSpec `json:"spec" protobuf:"bytes,2,name=spec"`
 }
 
 // ShuffleServerUpgradeStrategy defines upgrade strategy of shuffle servers.