Build docker image with hops hadoop distribution

Author: gibchikafa

Jenkinsfile

@@ -0,0 +1,37 @@
+pipeline {
+    agent {
+        label "local"
+    }
+    environment {
+        VERSION = readFile "${env.WORKSPACE}/version.txt"
+        BUILD_BRANCH = readFile "${env.WORKSPACE}/build_branch.txt"
+    }
+    stages {
+        stage("build and publish") {
+            agent {
+                label "local"
+            }
+            steps {
+              sh """
+                set -ex
+                docker login -u ${env.DOCKER_USER} -p ${env.DOCKER_PASS}
+                git checkout ${BUILD_BRANCH}
+                ./build_distribution.sh --spark-profile spark3 --hadoop-profile hadoop3.2
+                deploy/kubernetes/docker/build.sh --hadoop-version 3.2.0.13-EE-SNAPSHOT --registry docker.hops.works
+                RSS_VERSION=$(./mvnw help:evaluate -Dexpression=project.version 2>/dev/null | grep -v "INFO" | grep -v "WARNING" | tail -n 1)
+                mkdir -p /opt/repository/master/rss/${VERSION}/
+                cp  client/target/rss-client-{RSS_VERSION}.jar /opt/repository/master/rss/${VERSION}/
+              """
+            }
+        }
+    }
+
+    post {
+        success {
+            build job:'LC-Docker-imgs', parameters: [
+            string(name: 'image', value: "git"),
+            string(name: 'branch', value: "master")
+            ]
+        }
+    }
+}

build_branch.txt

@@ -0,0 +1 @@
+branch-0.9

deploy/kubernetes/docker/Dockerfile

@@ -1,19 +1,4 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
+# syntax=docker/dockerfile:experimental
 
 ARG BASE_IMAGE=uniffle-base:latest
 FROM ${BASE_IMAGE}
@@ -24,6 +9,7 @@ ARG RSS_VERSION
 ARG AUTHOR=unknown
 ARG GIT_BRANCH=unknown
 ARG GIT_COMMIT=unknown
+ARG HADOOP_FILE=hops-$HADOOP_VERSION.tgz
 
 LABEL author=${AUTHOR}
 LABEL git-branch=${GIT_BRANCH}
@@ -44,10 +30,10 @@ COPY rss-env.sh /data/rssadmin/rss/bin
 
 COPY start.sh /data/rssadmin/rss/bin
 
-COPY hadoop-${HADOOP_VERSION}.tar.gz /data/rssadmin
-RUN tar -zxvf /data/rssadmin/hadoop-${HADOOP_VERSION}.tar.gz -C /data/rssadmin
-RUN mv /data/rssadmin/hadoop-${HADOOP_VERSION} /data/rssadmin/hadoop
-RUN rm -rf /data/rssadmin/hadoop-${HADOOP_VERSION}.tar.gz
+COPY $HADOOP_FILE /data/rssadmin
+RUN tar -zxvf /data/rssadmin/$HADOOP_FILE -C /data/rssadmin
+RUN mv /data/rssadmin/hops-${HADOOP_VERSION} /data/rssadmin/hadoop
+RUN rm -rf /data/rssadmin/$HADOOP_FILE
 COPY hadoopconfig/ /data/rssadmin/hadoop/etc/hadoop
 
 ENV RSS_VERSION ${RSS_VERSION}

deploy/kubernetes/docker/build.sh

@@ -35,8 +35,7 @@ function exit_with_usage() {
 }
 
 REGISTRY="docker.io/library"
-HADOOP_VERSION=2.8.5
-HADOOP_SHORT_VERSION=$(echo $HADOOP_VERSION | awk -F "." '{print $1"."$2}')
+HADOOP_VERSION=3.2.0.13-SNAPSHOT
 AUTHOR=$(whoami)
 # If you are based in China, you could pass --apache-mirror <a_mirror_url> when building this.
 APACHE_MIRROR="https://dlcdn.apache.org"
@@ -92,6 +91,8 @@ while (( "$#" )); do
   shift
 done
 
+HADOOP_SHORT_VERSION=$(echo $HADOOP_VERSION | awk -F "." '{print $1"."$2}')
+
 if [ -z "$BASE_IMAGE" ]; then
   echo "start building base image: uniffle-base"
   docker build -t "uniffle-base:latest" \
@@ -101,14 +102,21 @@ else
   echo "using base image(${BASE_IMAGE}) to build rss server"
 fi
 
+if test -f "/root/.wgetrc"; then
+    wget https://nexus.hops.works/repository/hopshadoop/hops-$HADOOP_VERSION_EE.tgz
+else
+    wget https://repo.hops.works/master/hops-$HADOOP_VERSION.tgz
+fi
 
-HADOOP_FILE=hadoop-${HADOOP_VERSION}.tar.gz
-ARCHIVE_HADOOP_URL=https://archive.apache.org/dist/hadoop/core/hadoop-${HADOOP_VERSION}/${HADOOP_FILE}
-HADOOP_URL=${APACHE_MIRROR}/hadoop/core/hadoop-${HADOOP_VERSION}/${HADOOP_FILE}
-echo "HADOOP_URL is either ${HADOOP_URL} or ${ARCHIVE_HADOOP_URL}"
-if [ ! -e "$HADOOP_FILE" ]; \
-  then wget "${HADOOP_URL}" || wget "$ARCHIVE_HADOOP_URL"; \
-  else echo "${HADOOP_FILE} has been downloaded"; \
+HADOOP_FILE=hops-$HADOOP_VERSION.tgz
+if [ ! -e "$HADOOP_FILE" ]; then
+  if test -f "/root/.wgetrc"; then
+      wget https://nexus.hops.works/repository/hopshadoop/$HADOOP_FILE
+  else
+      wget https://repo.hops.works/master/$HADOOP_FILE
+  fi
+else
+  echo "${HADOOP_FILE} has been downloaded";
 fi
 
 RSS_DIR=../../..
@@ -126,17 +134,20 @@ cp "$RSS_DIR/$RSS_FILE" .
 
 GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
 GIT_COMMIT=$(git describe --dirty --always --tags | sed 's/-/./g')
-echo "image version: ${IMAGE_VERSION:=$RSS_VERSION-$GIT_COMMIT}"
 IMAGE=$REGISTRY/rss-server:$IMAGE_VERSION
+IMAGE_VERSION=$(cat $RSS_DIR/version.txt | awk '{$1=$1; print}' | sed '/^$/d')
+echo "image version: ${IMAGE_VERSION}"
+IMAGE=$REGISTRY/rss:$IMAGE_VERSION
 echo "building image: $IMAGE"
-docker build --network=host -t "$IMAGE" \
+DOCKER_BUILDKIT=1 docker build --network=host -t "$IMAGE" \
              --build-arg RSS_VERSION="$RSS_VERSION" \
              --build-arg HADOOP_VERSION="$HADOOP_VERSION" \
              --build-arg HADOOP_SHORT_VERSION="$HADOOP_SHORT_VERSION" \
              --build-arg AUTHOR="$AUTHOR" \
              --build-arg GIT_COMMIT="$GIT_COMMIT" \
              --build-arg GIT_BRANCH="$GIT_BRANCH" \
              --build-arg BASE_IMAGE="$BASE_IMAGE" \
+             --build-arg HADOOP_FILE="$HADOOP_FILE" \
              -f Dockerfile --no-cache .
 
 if [ x"${PUSH_IMAGE}" == x"true" ]; then

version.txt

@@ -0,0 +1 @@
+0.9.0