Build docker image with hops hadoop distribution

Author: gibchikafa

Jenkinsfile

@@ -0,0 +1,42 @@
+pipeline {
+    agent {
+        label "local"
+    }
+    environment {
+        VERSION = readFile "${env.WORKSPACE}/version.txt"
+        BUILD_BRANCH = readFile "${env.WORKSPACE}/build_branch.txt"
+    }
+    stages {
+        stage("build and publish") {
+            agent {
+                label "local"
+            }
+            steps {
+             withCredentials([usernamePassword(credentialsId: 'cred', passwordVariable: 'NEXUS_CREDS_PSW', usernameVariable: 'NEXUS_CREDS_USR')]) {
+                sh """
+                    set -ex
+                    echo "Building RSS version ${VERSION} on branch ${BUILD_BRANCH}"
+                    docker login -u ${NEXUS_CREDS_USR} -p ${NEXUS_CREDS_PSW} docker.hops.works
+                    git checkout ${BUILD_BRANCH}
+                    ./build_distribution.sh --spark3-profile spark3 --hadoop-profile hadoop3.2 --without-dashboard
+                    cd deploy/kubernetes/docker ||  exit
+                    ./build.sh --hadoop-version 3.2.0.13-EE-SNAPSHOT --registry docker.hops.works --nexus-user $NEXUS_CREDS_USR --nexus-password $NEXUS_CREDS_PSW
+                    cd ../../..
+
+                    mkdir -p /opt/repository/master/rss/${VERSION}/
+                    cp  client/target/rss-client-${VERSION}.jar /opt/repository/master/rss/${VERSION}/
+                """
+             }
+            }
+        }
+    }
+
+    post {
+        success {
+            build job:'Remote shuffle services', parameters: [
+            string(name: 'image', value: "rss"),
+            string(name: 'branch', value: "master")
+            ]
+        }
+    }
+}

build_branch.txt

@@ -0,0 +1 @@
+branch-0.9

build_distribution.sh

@@ -122,10 +122,12 @@ done
 
 cd $RSS_HOME
 
+set +u
 if [ -z "$JAVA_HOME" ]; then
   echo "Error: JAVA_HOME is not set, cannot proceed."
   exit -1
 fi
+set -u
 
 if [ $(command -v git) ]; then
   GITREV=$(git rev-parse --short HEAD 2>/dev/null || :)

deploy/kubernetes/docker/Dockerfile

@@ -1,19 +1,4 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
+# syntax=docker/dockerfile:experimental
 
 ARG BASE_IMAGE=uniffle-base:latest
 FROM ${BASE_IMAGE}
@@ -24,6 +9,7 @@ ARG RSS_VERSION
 ARG AUTHOR=unknown
 ARG GIT_BRANCH=unknown
 ARG GIT_COMMIT=unknown
+ARG HADOOP_FILE=hops-$HADOOP_VERSION.tgz
 
 LABEL author=${AUTHOR}
 LABEL git-branch=${GIT_BRANCH}
@@ -44,10 +30,11 @@ COPY rss-env.sh /data/rssadmin/rss/bin
 
 COPY start.sh /data/rssadmin/rss/bin
 
-COPY hadoop-${HADOOP_VERSION}.tar.gz /data/rssadmin
-RUN tar -zxvf /data/rssadmin/hadoop-${HADOOP_VERSION}.tar.gz -C /data/rssadmin
-RUN mv /data/rssadmin/hadoop-${HADOOP_VERSION} /data/rssadmin/hadoop
-RUN rm -rf /data/rssadmin/hadoop-${HADOOP_VERSION}.tar.gz
+COPY $HADOOP_FILE /data/rssadmin
+RUN mkdir -p /data/rssadmin/hops-${HADOOP_VERSION}
+RUN tar -zxvf /data/rssadmin/$HADOOP_FILE --strip-components=1 -C /data/rssadmin/hops-${HADOOP_VERSION}
+RUN mv /data/rssadmin/hops-${HADOOP_VERSION} /data/rssadmin/hadoop
+RUN rm -rf /data/rssadmin/$HADOOP_FILE
 COPY hadoopconfig/ /data/rssadmin/hadoop/etc/hadoop
 
 ENV RSS_VERSION ${RSS_VERSION}

deploy/kubernetes/docker/base/debian/Dockerfile

@@ -20,5 +20,5 @@ ARG BASE_IMAGE=eclipse-temurin:11-jdk
 FROM ${BASE_IMAGE}
 
 RUN apt-get update && \
-    apt-get install -y zlib1g zlib1g-dev lzop lsof netcat dnsutils less procps iputils-ping && \
+    apt-get install -y zlib1g zlib1g-dev lzop lsof netcat dnsutils less procps iputils-ping curl && \
     apt-get clean && rm -rf /var/lib/apt/lists/*

deploy/kubernetes/docker/build.sh

@@ -29,14 +29,14 @@ function exit_with_usage() {
   echo "+------------------------------------------------------------------------------------------------------+"
   echo "| ./build.sh [--hadoop-version <hadoop version>] [--registry <registry url>] [--author <author name>]  |"
   echo "|            [--base-os-distribution <os distribution>] [--base-image <base image url>]                |"
-  echo "|            [--push-image <true|false>] [--apache-mirror <apache mirror url>]                         |"
+  echo "|            [--push-image <true|false>] [--apache-mirror <apache mirror url>] [--nexus-user <nexus username>]
+                     [--nexus-password <nexus password>]  |"
   echo "+------------------------------------------------------------------------------------------------------+"
   exit 1
 }
 
 REGISTRY="docker.io/library"
-HADOOP_VERSION=2.8.5
-HADOOP_SHORT_VERSION=$(echo $HADOOP_VERSION | awk -F "." '{print $1"."$2}')
+HADOOP_VERSION=3.2.0.13-SNAPSHOT
 AUTHOR=$(whoami)
 # If you are based in China, you could pass --apache-mirror <a_mirror_url> when building this.
 APACHE_MIRROR="https://dlcdn.apache.org"
@@ -70,6 +70,14 @@ while (( "$#" )); do
       PUSH_IMAGE="$2"
       shift
       ;;
+    --nexus-user)
+      NEXUS_USER="$2"
+      shift
+      ;;
+    --nexus-password)
+      NEXUS_PASSWORD="$2"
+      shift
+      ;;
     --help)
       exit_with_usage
       ;;
@@ -92,6 +100,8 @@ while (( "$#" )); do
   shift
 done
 
+HADOOP_SHORT_VERSION=$(echo $HADOOP_VERSION | awk -F "." '{print $1"."$2}')
+
 if [ -z "$BASE_IMAGE" ]; then
   echo "start building base image: uniffle-base"
   docker build -t "uniffle-base:latest" \
@@ -101,14 +111,15 @@ else
   echo "using base image(${BASE_IMAGE}) to build rss server"
 fi
 
-
-HADOOP_FILE=hadoop-${HADOOP_VERSION}.tar.gz
-ARCHIVE_HADOOP_URL=https://archive.apache.org/dist/hadoop/core/hadoop-${HADOOP_VERSION}/${HADOOP_FILE}
-HADOOP_URL=${APACHE_MIRROR}/hadoop/core/hadoop-${HADOOP_VERSION}/${HADOOP_FILE}
-echo "HADOOP_URL is either ${HADOOP_URL} or ${ARCHIVE_HADOOP_URL}"
-if [ ! -e "$HADOOP_FILE" ]; \
-  then wget "${HADOOP_URL}" || wget "$ARCHIVE_HADOOP_URL"; \
-  else echo "${HADOOP_FILE} has been downloaded"; \
+HADOOP_FILE=hops-$HADOOP_VERSION.tgz
+if [ ! -e "$HADOOP_FILE" ]; then
+  if [[ -n "$NEXUS_USER" ]] && [[ -n "$NEXUS_PASSWORD" ]] ; then
+      wget --user $NEXUS_USER --password $NEXUS_PASSWORD https://nexus.hops.works/repository/hopshadoop/$HADOOP_FILE
+  else
+      wget https://repo.hops.works/master/$HADOOP_FILE
+  fi
+else
+  echo "${HADOOP_FILE} has been downloaded";
 fi
 
 RSS_DIR=../../..
@@ -126,17 +137,21 @@ cp "$RSS_DIR/$RSS_FILE" .
 
 GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
 GIT_COMMIT=$(git describe --dirty --always --tags | sed 's/-/./g')
-echo "image version: ${IMAGE_VERSION:=$RSS_VERSION-$GIT_COMMIT}"
+IMAGE_VERSION=$(cat $RSS_DIR/version.txt | awk '{$1=$1; print}' | sed '/^$/d')
 IMAGE=$REGISTRY/rss-server:$IMAGE_VERSION
+echo "image version: ${IMAGE_VERSION}"
+IMAGE=$REGISTRY/rss:$IMAGE_VERSION
 echo "building image: $IMAGE"
-docker build --network=host -t "$IMAGE" \
+
+DOCKER_BUILDKIT=1 docker build --network=host -t "$IMAGE" \
              --build-arg RSS_VERSION="$RSS_VERSION" \
              --build-arg HADOOP_VERSION="$HADOOP_VERSION" \
              --build-arg HADOOP_SHORT_VERSION="$HADOOP_SHORT_VERSION" \
              --build-arg AUTHOR="$AUTHOR" \
              --build-arg GIT_COMMIT="$GIT_COMMIT" \
              --build-arg GIT_BRANCH="$GIT_BRANCH" \
              --build-arg BASE_IMAGE="$BASE_IMAGE" \
+             --build-arg HADOOP_FILE="$HADOOP_FILE" \
              -f Dockerfile --no-cache .
 
 if [ x"${PUSH_IMAGE}" == x"true" ]; then

version.txt

@@ -0,0 +1 @@
+0.9.0