Jenkinsfile.oke

// Copyright (c) 2023, 2024, Oracle and/or its affiliates.
// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
//

CRON_SETTINGS_MAIN = '''H 3 * * * % MAVEN_PROFILE_NAME=oke-gate;CLUSTER_NAME=seqone;PARALLEL_RUN=false
                   H 2 * * * % MAVEN_PROFILE_NAME=oke-parallel;CLUSTER_NAME=parone;PARALLEL_RUN=true'''

CRON_SETTINGS_42 = '''H 3 * * * % MAVEN_PROFILE_NAME=oke-gate;CLUSTER_NAME=seq42one;PARALLEL_RUN=false
                   H 2 * * * % MAVEN_PROFILE_NAME=oke-parallel;CLUSTER_NAME=par42one;PARALLEL_RUN=true'''

CRON_SETTINGS = "${env.JOB_NAME == 'wko-oke-nightly' ? CRON_SETTINGS_MAIN : CRON_SETTINGS_42}"

pipeline {
    agent { label 'large' }
    options {
        timeout(time: 1800, unit: 'MINUTES')
        //disableConcurrentBuilds()
    }
    triggers {
        // timer trigger for "nightly build"
        parameterizedCron(env.JOB_NAME == 'wko-oke-dev' ?
                '' : CRON_SETTINGS)
    }
    tools {
        maven 'maven-3.8.7'
        jdk 'jdk21'
    }

    environment {
        ocir_host = "${env.WKT_OCIR_HOST}"
        wko_tenancy = "${env.WKT_TENANCY}"
        wko_images_rep = '${wko_tenancy}/wkt/infra'
        wko_region = "${env.WKT_REGION}"
        compartment_id = "${env.WKT_TEST_COMPARTMENT_ID}"
        ocir_creds = 'wkt-ocir-creds'
        wko_files_bucket_token = 'wko-system-test-files-bucket-par-token'

        oci_profile = 'DEFAULT'
        jenkins_user = sh(returnStdout: true, script: "id -nu").trim()
        jenkins_home_directory = sh(returnStdout: true, script: "getent passwd ${jenkins_user} | cut -d: -f6").trim()
        
        oci_config_file_creds = 'wkt-oci-config-file'
        oci_signing_key_creds = 'wkt-oci-signing-key'

        terraform_property_file = 'tvarsFile'
        compartmentName='test'

        wkobuild_cert_fingerprint = 'wkotestcert_fingerprint'
        wkobuild_user_ocid = 'wkotest-user-ocid'

        wkotest_ssh_pubcert = "${WORKSPACE}/.ssh/wkotestkey.pub"
        wkotest_ssh_pk = "${WORKSPACE}/.ssh/wkotestkey"

        OKE_CLUSTER='true'

        outdir = "${WORKSPACE}/staging"
        result_root = "${outdir}/wl_k8s_test_results"
        pv_root = "${outdir}/k8s-pvroot"

        OCI_PROP_FILE="${WORKSPACE}/terraform/oci.prop"

        start_time = sh(script: 'date +"%Y-%m-%d %H:%M:%S"', returnStdout: true).trim()
        wle_download_url="https://github.com/oracle/weblogic-logging-exporter/releases/latest"
        kubeconfig_file = "${WORKSPACE}/terraform/${CLUSTER_NAME}_kubeconfig"
        availability_domain = "${env.JOB_NAME == 'wko-oke-nightly-parallel' ? 'mFEn:PHX-AD-1' : 'mFEn:PHX-AD-1'}"
        //oke_run = "${env.JOB_NAME == 'wko-oke-nightly' ? 'okeint' : 'okeintdev'}"
        oke_run = "okemodule"

    }

    parameters {

        string(name: 'IT_TEST',
               description: 'Comma separated list of individual It test classes to be run e.g., ItParameterizedDomain, ItMiiUpdateDomainConfig, ItMiiDynamicUpdate*, ItMiiMultiMode',
               defaultValue: ''
        )

        booleanParam(name: 'PARALLEL_RUN',
                             description: 'Runs tests in parallel. Default is false.',
                             defaultValue: false
                )

        string(name: 'OKE_KUBE_VERSION',
               description: 'kube version for oke cluster',
               defaultValue: '1.29.1'
        )
        string(name: 'IMAGE_ID',
                       description: 'oci image id for node pool, find image OCID for your region from https://docs.oracle.com/iaas/images/',
                       //defaultValue OKE1.26.2: 'ocid1.image.oc1.phx.aaaaaaaaaizmtmozeudeeuq7o5ir7dkl2bkxbbb3tgomshqbqn6jpomrsjza'
                       //1.27.2 oke defaultValue: 'ocid1.image.oc1.phx.aaaaaaaaypr5r5drojwytghw6e6mvpjsscrnkuwtmqlmvmix7kjb2zcnc7wa'
                       defaultValue: 'ocid1.image.oc1.phx.aaaaaaaa22u45gr3ikxnc6rius2qil2kz5k3e7p476c4usr6qnvql4l5dxea'
                       
                )
        string(name: 'KUBECTL_VERSION',
                       description: 'kubectl version',
                       defaultValue: '1.26.2'
                )
        string(name: 'HELM_VERSION',
               description: 'Helm version',
               defaultValue: '3.11.2'
        )
        choice(name: 'ISTIO_VERSION',
               description: 'Istio version',
               choices: [
	           '1.23.0',
                   '1.17.2',
                   '1.16.1',
                   '1.13.2',
                   '1.12.6',
                   '1.11.1',
                   '1.10.4',
                   '1.9.9'
               ]
        )

                choice(name: 'MAVEN_PROFILE_NAME',
                        description: 'Profile to use in mvn command to run the tests.  Possible values are oke-gate,oke-parallel. Refer to weblogic-kubernetes-operator/integration-tests/pom.xml on the branch.',
                        choices: [
                                'oke-gate',
                                'oke-parallel'
                        ]
                )
        string(name: 'NUMBER_OF_THREADS',
               description: 'Number of threads to run the classes in parallel, default is 2.',
               defaultValue: "2"
        )
        string(name: 'CLUSTER_NAME',
                       description: 'OKE cluster name',
                       defaultValue: "seqone"
                )
        string(name: 'WDT_DOWNLOAD_URL',
               description: 'URL to download WDT.',
               //defaultValue: 'https://github.com/oracle/weblogic-deploy-tooling/releases/latest'
        )
        string(name: 'WIT_DOWNLOAD_URL',
               description: 'URL to download WIT.',
               //defaultValue: 'https://github.com/oracle/weblogic-image-tool/releases/latest'
        )
        string(name: 'REPO_REGISTRY',
               description: '',
               defaultValue: "${env.WKT_OCIR_HOST}"
        )
        string(name: 'NODE_SHAPE',
                       description: '',
                       defaultValue: "VM.Standard.E4.Flex"
        )
        string(name: 'MOUNT_TARGET_OCID',
                               description: 'only for debug runs on wko-oke-dev',
                               defaultValue: "ocid1.mounttarget.oc1.phx.aaaaaby27vhqpci5obuhqllqojxwiotqnb4c2ylefuzqaaaa"
                )
        string(name: 'BRANCH',
                       description: '',
                       defaultValue: "main"
        )
        choice(name: 'BASE_IMAGES_REPO',
               choices: ["${env.WKT_OCIR_HOST}", 'container-registry.oracle.com'],
               description: 'Repository to pull the base images. Make sure to modify the image names if you are modifying this parameter value.'
        )
        string(name: 'TEST_IMAGES_REPO',
               description: '',
               defaultValue: "${env.WKT_OCIR_HOST}"
        )
        string(name: 'WEBLOGIC_IMAGE_NAME',
               description: 'WebLogic base image name. Default is the image name in OCIR. Use middleware/weblogic for OCR.',
               defaultValue: 'test-images/weblogic'
        )
        string(name: 'WEBLOGIC_IMAGE_TAG',
               description: '12.2.1.4,  12.2.1.4-dev(12.2.1.4-dev-ol7) , 12.2.1.4-slim(12.2.1.4-slim-ol7), 12.2.1.4-ol8, 12.2.1.4-dev-ol8, 12.2.1.4-slim-ol8, 14.1.1.0-11-ol7, 14.1.1.0-dev-11-ol7, 14.1.1.0-slim-11-ol7, 14.1.1.0-8-ol7, 14.1.1.0-dev-8-ol7, 14.1.1.0-slim-8-ol7, 14.1.1.0-11-ol8, 14.1.1.0-dev-11-ol8, 14.1.1.0-slim-11-ol8, 14.1.1.0-8-ol8, 14.1.1.0-dev-8-ol8, 14.1.1.0-slim-8-ol8',
               defaultValue: '12.2.1.4'
        )
        string(name: 'FMWINFRA_IMAGE_NAME',
               description: 'FWM Infra image name. Default is the image name in OCIR. Use middleware/fmw-infrastructure for OCR.',
               defaultValue: 'test-images/fmw-infrastructure'
        )
        string(name: 'FMWINFRA_IMAGE_TAG',
               description: 'FWM Infra image tag',
               defaultValue: '12.2.1.4'
        )
        string(name: 'DB_IMAGE_NAME',
               description: 'Oracle DB image name. Default is the image name in OCIR, use database/enterprise for OCR.',
               defaultValue: 'test-images/database/enterprise'
        )
        string(name: 'DB_IMAGE_TAG',
               description: 'Oracle DB image tag',
               defaultValue: '12.2.0.1-slim'
        )
        string(name: 'MONITORING_EXPORTER_BRANCH',
               description: '',
               defaultValue: 'main'
        )
        string(name: 'MONITORING_EXPORTER_WEBAPP_VERSION',
               description: '',
               defaultValue: '2.1.3'
        )
        string(name: 'PROMETHEUS_CHART_VERSION',
               description: '',
               defaultValue: '17.0.0'
        )
        string(name: 'GRAFANA_CHART_VERSION',
               description: '',
               defaultValue: '6.44.11'
        )
        booleanParam(name: 'COLLECT_LOGS_ON_SUCCESS',
                     description: 'Collect logs for successful runs. Default is false.',
                     defaultValue: false
        )
        string(name: 'REMOTECONSOLE_VERSION',
               description: 'RemoteConsole version.',
               defaultValue: '2.4.7'
        )
    }


    stages {
        stage('Filter unwanted branches') {
            stages {
                stage('Workaround JENKINS-41929 Parameters bug') {
                    steps {
                        echo 'Initialize parameters as environment variables due to https://issues.jenkins-ci.org/browse/JENKINS-41929'
                        evaluate """${def script = ""; params.each { k, v -> script += "env.${k} = '''${v}'''\n" }; return script}"""
                    }
                }
                stage ('Echo environment') {
                    environment {
                        runtime_path = "${WORKSPACE}/bin:${PATH}"
                    }
                    steps {
                        sh '''
                            env|sort
                            java -version
                            mvn --version
                            python --version
                            docker version
                            ulimit -a
                            ulimit -aH
                        '''
                    }
                }
                stage('Make Workspace bin directory') {
                    steps {
                        sh "mkdir -m777 -p ${WORKSPACE}/bin"
                    }
                }
                stage('Install Helm') {
                    environment {
                        runtime_path = "${WORKSPACE}/bin:${PATH}"
                    }
                    steps {
                        sh '''
                            export PATH=${runtime_path}
                            oci os object get --namespace=${wko_tenancy} --bucket-name=wko-system-test-files \
                                --name=helm/helm-v${HELM_VERSION}.tar.gz --file=helm.tar.gz \
                                --auth=instance_principal
                            tar zxf helm.tar.gz
                            mv linux-amd64/helm ${WORKSPACE}/bin/helm
                            rm -rf linux-amd64
                            helm version
                        '''
                    }
                }

                stage ('Install kubectl') {
                    environment {
                        runtime_path = "${WORKSPACE}/bin:${PATH}"
                    }
                    steps {
                        sh '''
                            export PATH=${runtime_path}
                            oci os object get --namespace=${wko_tenancy} --bucket-name=wko-system-test-files \
                                                            --name=kubectl/kubectl-v${KUBECTL_VERSION} --file=${WORKSPACE}/bin/kubectl \
                                                            --auth=instance_principal
                            chmod +x ${WORKSPACE}/bin/kubectl
                            kubectl version --client=true
                        '''
                    }
                }

                // Creating terraform property file.

                stage('Setup terraform property file  ') {
                    environment {
                        OKE_CLUSTER_NAME="${CLUSTER_NAME}"
                        NODE_SHAPE="${NODE_SHAPE}"
                        IMAGE_ID="${IMAGE_ID}"
                        OCI_PROP_FILE="${WORKSPACE}/terraform/oci.prop"
                        FINGERPRINT = 'wkobuild_cert_fingerprint'
                        USER_OCID = 'wkobuild_user_ocid'
                        WKOBUILD_PUBLIC_CERT = "${wkotest_ssh_pubcert}"
                        runtime_path = "${WORKSPACE}/bin:${PATH}"

                    }
                    steps {

                        withCredentials([file(credentialsId: "${oci_config_file_creds}", variable: 'OCI_CONFIG_FILE'),
                                                        file(credentialsId: "${oci_signing_key_creds}", variable: 'OCI_SIGNING_KEY')]) {
                                           sh "mkdir -p ${jenkins_home_directory}/.oci"
                                           sh "sed -e \"s:__jenkins_home__:${jenkins_home_directory}:g\" \"\$OCI_CONFIG_FILE\" > ${jenkins_home_directory}/.oci/config"
                                           sh "cp \"\$OCI_SIGNING_KEY\" ${jenkins_home_directory}/.oci/oci-signing-key.pem"
                                       }


                        sh '''
                        TENANCY_OCID=$(grep -Po '(?<=tenancy=).*' ${jenkins_home_directory}/.oci/config)
                        USER_OCID=$(grep -Po '(?<=user=).*' ${jenkins_home_directory}/.oci/config)
                        FINGERPRINT=$(grep -Po '(?<=fingerprint=).*' ${jenkins_home_directory}/.oci/config)
                        compname="wkt"
                        wkt_compartment_ocid=$(oci iam compartment list --compartment-id-in-subtree true --all | jq --arg compname "$compname" '.data[] | select(."name"==$compname)' | jq -r ."id")
                        vcn_ocid=$(oci network vcn list --compartment-id=${wkt_compartment_ocid}| jq -r '.data[] | .id')
                        pub_subnet_ocid=$(oci network subnet list --compartment-id=${wkt_compartment_ocid}  | jq '.data[] | select(."display-name" == "Public-Subnet-wktiso1")' | jq -r ."id")
                        private_subnet_ocid=$(oci network subnet list --compartment-id=${wkt_compartment_ocid}  | jq '.data[] | select(."display-name" == "Private-Subnet-wktiso1")' | jq -r ."id")
                        export WKT_TENANCY_OCID=${TENANCY_OCID}
                        export WKT_USER_OCID=${USER_OCID}
                        mkdir -p ${WORKSPACE}/terraform
                        mkdir -p ${WORKSPACE}/oci
                        cat ${jenkins_home_directory}/.oci/config
                        mkdir -p ${WORKSPACE}/.ssh
                        cd ${WORKSPACE}/.ssh
                        ssh-keygen -b 2048 -t rsa -q  -f wkotestkey -N ""
                        chmod 600 ${wkotest_ssh_pk}

                        COMPARTMENT_OCID=${compartment_id}
                        export OCI_CLI_SUPPRESS_FILE_PERMISSIONS_WARNING=True

                        echo 'Set env vars needed for integration tests...'


                        export OKE_CLUSTER="true"

                        VCN_CIDR_PREFIX="10.196"
                        export VCN_CIDR_PREFIX=${VCN_CIDR_PREFIX}

                        export OKE_CLUSTER_NAME=${OKE_CLUSTER_NAME}
                        export NODE_SHAPE=${NODE_SHAPE}
                        export IMAGE_ID=${IMAGE_ID}

                        export OCI_PROP_FILE=${OCI_PROP_FILE}

                        #export IMAGE_TAG_WEBLOGIC="12.2.1.4"
                        #export IMAGE_TAG_FMWINFRA="12.2.1.4"
                        ssh_pubkey=`cat ${wkotest_ssh_pubcert}`
                        ssh_pk=`cat ${wkotest_ssh_pk}`

                        #################
                        echo "Generating property file oci.prop for terraform scripts"

cat > $OCI_PROP_FILE <<EOF

user.ocid=${WKT_USER_OCID}
okeclustername=${OKE_CLUSTER_NAME}
tfvars.filename=${OKE_CLUSTER_NAME}
region=${wko_region}
tenancy.ocid=${WKT_TENANCY_OCID}
compartment.ocid=${COMPARTMENT_OCID}
sub.comp.ocid=${wkt_compartment_ocid}
compartment.name=${compartmentName}
ociapi.pubkey.fingerprint=${FINGERPRINT}
ocipk.path=${jenkins_home_directory}/.oci/oci-signing-key.pem
vcn.cidr.prefix=${VCN_CIDR_PREFIX}
vcn.cidr=${VCN_CIDR_PREFIX}.1.0/24
vcn.ocid=${vcn_ocid}
pub.subnet.ocid=${pub_subnet_ocid}
private.subnet.ocid=${private_subnet_ocid}
nodepool.shape=${NODE_SHAPE}
mounttarget.ocid=${MOUNT_TARGET_OCID}
nodepool.imagename=${IMAGE_ID}
k8s.version=v${OKE_KUBE_VERSION}
nodepool.ssh.pubkey=${ssh_pubkey}
nodepool.ssh.pk=${ssh_pk}
nodepool.ssh.pubkeypath=${wkotest_ssh_pubcert}
nodepool.ssh.pkpath=${wkotest_ssh_pk}
terraform.installdir=${WORKSPACE}/terraform/terraforminstall
EOF

                        ##################
                        echo "prop files "
                        cat $OCI_PROP_FILE
                        mkdir -p ${WORKSPACE}/terraform/terraforminstall
                        '''

                    }
                }
                stage('Build WebLogic Kubernetes Operator') {

                    steps {
                        withMaven(globalMavenSettingsConfig: 'wkt-maven-settings-xml', publisherStrategy: 'EXPLICIT') {
                                                    sh "mvn -DtrimStackTrace=false clean install"
                        }
                    }
                }



                stage('Preparing Integration Test Environment') {
                    steps {
                        sh 'mkdir -m777 -p ${result_root}'
                        echo "Results will be in ${result_root}"
                        sh 'mkdir -m777 -p ${pv_root}'
                        echo "Persistent volume files, if any, will be in ${pv_root}"
                    }
                }

                stage('Create OKE cluster') {
                    environment {
                        runtime_path = "${WORKSPACE}/bin:${PATH}"
                        
                        OCI_PROP_FILE="${WORKSPACE}/terraform/oci.prop"
                        CLUSTER_NAME="${CLUSTER_NAME}"
                        KUBECONFIG ="${WORKSPACE}/terraform/${CLUSTER_NAME}_kubeconfig"
                        terraform_script_dir_name = "${WORKSPACE}/integration-tests/src/test/resources/oke/terraform/${oke_run}"

                    }
                    steps {

                        sh '''
                            export PATH=${runtime_path}
                            export KUBECONFIG=${KUBECONFIG}

                            export OCI_CLI_CONFIG_FILE=${jenkins_home_directory}/.oci/config
                            export OCI_CLI_PROFILE=${oci_profile}

                            echo 'Create a OKE cluster ${CLUSTER_NAME}'

                            cp -rf ${terraform_script_dir_name}/* ${WORKSPACE}/terraform/.

                            chmod 777 ${WORKSPACE}/terraform/*.sh
                            mkdir -p ${WORKSPACE}/terraform/terraforminstall
                            sh ${WORKSPACE}/terraform/oke.create.sh ${OCI_PROP_FILE} ${WORKSPACE}/terraform

                            clusterIP=$(oci ce cluster list --compartment-id=${compartment_id} | jq '.data[]  | select(."name" == "'"${CLUSTER_NAME}"'" and (."lifecycle-state" == "ACTIVE"))' | jq ' ."endpoints" | ."public-endpoint"')

                            echo "clusterIp : $clusterIP"
                            clusterPublicIP=${clusterIP:1:-6}
                            echo " clusterPublicIP : ${clusterPublicIP}"
                            export NO_PROXY=$NO_PROXY,${clusterPublicIP}
                            echo "NO_PROXY:" $NO_PROXY


                            export SERVICE_ACCOUNT_NAME=${CLUSTER_NAME}-sa
                            ${KUBERNETES_CLI:-kubectl} -n kube-system create serviceaccount ${SERVICE_ACCOUNT_NAME}
                            ${KUBERNETES_CLI:-kubectl} create clusterrolebinding add-on-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:${SERVICE_ACCOUNT_NAME}
                            sed -i 's@%SERVICE_ACCOUNT_NAME%@'"${SERVICE_ACCOUNT_NAME}"'@' ${WORKSPACE}/integration-tests/src/test/resources/oke/secret.yaml
                            ${KUBERNETES_CLI:-kubectl} apply -f ${WORKSPACE}/integration-tests/src/test/resources/oke/secret.yaml

                            TOKEN=`${KUBERNETES_CLI:-kubectl} -n kube-system get secret secret-${SERVICE_ACCOUNT_NAME} -o jsonpath='{.data.token}'| base64 --decode`
                            ${KUBERNETES_CLI:-kubectl} config set-credentials ${SERVICE_ACCOUNT_NAME} --token=$TOKEN
                            ${KUBERNETES_CLI:-kubectl} config set-context --current --user=${SERVICE_ACCOUNT_NAME}
                        '''
                    }
                }

                stage('Setup OFSS') {
                    environment {
                        runtime_path = "${WORKSPACE}/bin:${PATH}:${WORKSPACE}/terraform/terraforminstall"
                        OCI_PROP_FILE = "${WORKSPACE}/terraform/oci.prop"

                        clusterName = "${CLUSTER_NAME}"
                        terraform_script_dir_name = "${WORKSPACE}/integration-tests/src/test/resources/oke/terraform/"


                    }
                    steps {
                        sh '''
                        export KUBECONFIG=${kubeconfig_file}
                        export OCI_CLI_CONFIG_FILE=${jenkins_home_directory}/.oci/config
                        export OCI_CLI_PROFILE=${oci_profile}
                        export PATH=${runtime_path}
                        compartment_ocid=${compartment_id}
                        echo "creating storage class to setup OFSS ..."
                        echo "getting MountTarget ID"
                        mount_target_id=${MOUNT_TARGET_OCID}

                        clusterIP=$(oci ce cluster list --compartment-id=${compartment_id} | jq '.data[]  | select(."name" == "'"${CLUSTER_NAME}"'" and (."lifecycle-state" == "ACTIVE"))' | jq ' ."endpoints" | ."public-endpoint"')
                        echo "clusterIp : $clusterIP"
                        clusterPublicIP=${clusterIP:1:-6}
                        echo " clusterPublicIP : ${clusterPublicIP}"
                        export NO_PROXY=$NO_PROXY,${clusterPublicIP}
                        echo "NO_PROXY:" $NO_PROXY
                        echo "Creating Storage Class to mount OFSS"
                        export MOUNT_TARGET_ID=${mount_target_id}
                        export AVAILABILITY_DOMAIN=${availability_domain}
                        sed -i 's@%AVAILABILITY_DOMAIN%@'"$AVAILABILITY_DOMAIN"'@' ${WORKSPACE}/integration-tests/src/test/resources/oke/ofssnew.yaml
                        sed -i 's@%MOUNT_TARGET_ID%@'"$MOUNT_TARGET_ID"'@' ${WORKSPACE}/integration-tests/src/test/resources/oke/ofssnew.yaml
                        kubectl apply -f ${WORKSPACE}/integration-tests/src/test/resources/oke/ofssnew.yaml
                        kubectl delete csiDriver fss.csi.oraclecloud.com
                        kubectl create -f ${WORKSPACE}/integration-tests/src/test/resources/oke/fss_csi_driver.yaml

                        '''
                    }
                }

                stage('Run integration tests') {
                    environment {
                        runtime_path = "${WORKSPACE}/bin:${PATH}"
                        clusterName = "${CLUSTER_NAME}"
                        FSS_DIR = "${env.JOB_NAME == 'wko-oke-nightly' ? '/${clusterName}oketest1,/${clusterName}oketest2' : '/${clusterName}oketest1,/${clusterName}oketest2'}"


                    }
                    steps {
                         script {
                             def res = 0
                             currentBuild.description = "${GIT_BRANCH} ${MAVEN_PROFILE_NAME} ${CLUSTER_NAME}"
                             res = sh(script: '''
                                     if [ -z "${IT_TEST}" ] && [ "${MAVEN_PROFILE_NAME}" = "integration-tests" ]; then
                                        echo 'ERROR: All tests cannot be run with integration-tests profile'
                                        exit 1
                                     fi
                                 ''', returnStatus: true)
                             if (res != 0 ) {
                                 currentBuild.result = 'ABORTED'
                                 error('Profile/ItTests Validation Failed')
                             }
                        }
                        withCredentials([
                             string(credentialsId: "${wko_files_bucket_token}", variable: 'WKO_BUCKET_TOKEN')
                        ]) {
                            sh '''
                                if [ -z $WIT_DOWNLOAD_URL ]; then
                                    WIT_DOWNLOAD_URL="https://objectstorage.${wko_region}.oraclecloud.com/p/${WKO_BUCKET_TOKEN}/n/${wko_tenancy}/b/wko-system-test-files/o/imagetool-main.zip"
                                fi
                                if [ -z $WDT_DOWNLOAD_URL ]; then
                                    WDT_DOWNLOAD_URL="https://objectstorage.${wko_region}.oraclecloud.com/p/${WKO_BUCKET_TOKEN}/n/${wko_tenancy}/b/wko-system-test-files/o/weblogic-deploy-main.zip"
                                fi

                                export PATH=${runtime_path}
                                export OCI_CLI_CONFIG_FILE=${jenkins_home_directory}/.oci/config
                                export OCI_CLI_PROFILE=${oci_profile}
                                compartment_ocid=${compartment_id}
                                mkdir -m777 -p "${WORKSPACE}/.mvn"
                                touch ${WORKSPACE}/.mvn/maven.config
                                export KUBECONFIG=${kubeconfig_file}

                                clusterIP=$(oci ce cluster list --compartment-id=${compartment_id} | jq '.data[]  | select(."name" == "'"${CLUSTER_NAME}"'" and (."lifecycle-state" == "ACTIVE"))' | jq ' ."endpoints" | ."public-endpoint"')
                                echo "clusterIp : $clusterIP"
                                clusterPublicIP=${clusterIP:1:-6}
                                echo " clusterPublicIP : ${clusterPublicIP}"
                                export NO_PROXY=$NO_PROXY,${clusterPublicIP}
                                echo "NO_PROXY:" $NO_PROXY
                                NODE_IP=`kubectl get nodes -o wide| awk '{print $7}'| tail -n+3`
                                if [ -z "${NODE_IP}" ]; then
                                        echo "retry get node ip ";
                                  sleep 15;
                                  NODE_IP=`kubectl get nodes -o wide| awk '{print $7}'| tail -n+3`
                                fi

                                export K8S_NODEPORT_HOST=${NODE_IP}
                                export AVAILABILITY_DOMAIN=${availability_domain}
                                NODE_IP=`kubectl get nodes -o wide| awk '{print $7}'| head -n2 | tail -n1`
                                echo "second node external IP ${NODE_IP}"
                                export NODE_IP=${NODE_IP}

                                mt_privateip_id=`oci fs mount-target get --mount-target-id=${MOUNT_TARGET_OCID} | jq -r '.data| ."private-ip-ids"[]'`

                                # Check if the mt_privateip_id is an array
                                if [ "$(declare -p mt_privateip_id 2>/dev/null | grep -o 'declare -a')" == "declare -a" ]; then
                                    # Select first
                                    mt_private_ip=`oci network private-ip get --private-ip-id "${mt_privateip_id[0]}" | jq -r '.data | ."ip-address"'`
                                else
                                    mt_private_ip=`oci network private-ip get --private-ip-id "${mt_privateip_id}" | jq -r '.data | ."ip-address"'`
                                fi



                                if [ -z "${mt_private_ip}" ]; then
                                   echo "Mount Target was not setup properly , clean up Kubernetes cluster"
                                   sh ${WORKSPACE}/terraform/oke.delete.sh ${OCI_PROP_FILE} ${WORKSPACE}/terraform ${AVAILABILITY_DOMAIN}
                                   exit -1
                                fi
                                export NFS_SERVER=$mt_private_ip
                                export FSS_DIR=${FSS_DIR}
                                echo "Using NFS Server ${NFS_SERVER}"

                                if [ -n "${IT_TEST}" ]; then
                                    echo 'Overriding MAVEN_PROFILE_NAME to integration-test when running individual test(s)'
                                    MAVEN_PROFILE_NAME="integration-tests"
                                    echo "-Dit.test=\"${IT_TEST}\"" >> ${WORKSPACE}/.mvn/maven.config
                                fi

                                echo "-Dwko.it.wle.download.url=\"${wle_download_url}\""                                     >> ${WORKSPACE}/.mvn/maven.config
                                echo "-Dwko.it.result.root=\"${result_root}\""                                               >> ${WORKSPACE}/.mvn/maven.config
                                echo "-Dwko.it.pv.root=\"${pv_root}\""                                                       >> ${WORKSPACE}/.mvn/maven.config
                                echo "-Dwko.it.k8s.nodeport.host=\"${K8S_NODEPORT_HOST}\""                                   >> ${WORKSPACE}/.mvn/maven.config
                                echo "-Dwko.it.nfs.server=\"${NFS_SERVER}\""                                          >> ${WORKSPACE}/.mvn/maven.config
                                echo "-Dwko.it.istio.version=\"${ISTIO_VERSION}\""                                           >> ${WORKSPACE}/.mvn/maven.config
                                echo "-DPARALLEL_CLASSES=\"${PARALLEL_RUN}\""                                                >> ${WORKSPACE}/.mvn/maven.config
                                echo "-DNUMBER_OF_THREADS=\"${NUMBER_OF_THREADS}\""                                          >> ${WORKSPACE}/.mvn/maven.config
                                echo "-Dwko.it.wdt.download.url=\"${WDT_DOWNLOAD_URL}\""                                     >> ${WORKSPACE}/.mvn/maven.config
                                echo "-Dwko.it.wit.download.url=\"${WIT_DOWNLOAD_URL}\""                                     >> ${WORKSPACE}/.mvn/maven.config
                                echo "-Dwko.it.base.images.repo=\"${BASE_IMAGES_REPO}\""                                     >> ${WORKSPACE}/.mvn/maven.config
                                echo "-Dwko.it.base.images.tenancy=\"${wko_tenancy}\""                                       >> ${WORKSPACE}/.mvn/maven.config
                                echo "-Dwko.it.test.images.repo=\"${TEST_IMAGES_REPO}\""                                     >> ${WORKSPACE}/.mvn/maven.config
                                echo "-Dwko.it.test.images.tenancy=\"${wko_tenancy}\""                                       >> ${WORKSPACE}/.mvn/maven.config
                                echo "-Dwko.it.weblogic.image.name=\"${WEBLOGIC_IMAGE_NAME}\""                               >> ${WORKSPACE}/.mvn/maven.config
                                echo "-Dwko.it.weblogic.image.tag=\"${WEBLOGIC_IMAGE_TAG}\""                                 >> ${WORKSPACE}/.mvn/maven.config
                                echo "-Dwko.it.fmwinfra.image.name=\"${FMWINFRA_IMAGE_NAME}\""                               >> ${WORKSPACE}/.mvn/maven.config
                                echo "-Dwko.it.fmwinfra.image.tag=\"${FMWINFRA_IMAGE_TAG}\""                                 >> ${WORKSPACE}/.mvn/maven.config
                                echo "-Dwko.it.db.image.name=\"${DB_IMAGE_NAME}\""                                           >> ${WORKSPACE}/.mvn/maven.config
                                echo "-Dwko.it.db.image.tag=\"${DB_IMAGE_TAG}\""                                             >> ${WORKSPACE}/.mvn/maven.config
                                echo "-Dwko.it.monitoring.exporter.branch=\"${MONITORING_EXPORTER_BRANCH}\""                 >> ${WORKSPACE}/.mvn/maven.config
                                echo "-Dwko.it.monitoring.exporter.webapp.version=\"${MONITORING_EXPORTER_WEBAPP_VERSION}\"" >> ${WORKSPACE}/.mvn/maven.config
                                echo "-Dwko.it.prometheus.chart.version=\"${PROMETHEUS_CHART_VERSION}\""                     >> ${WORKSPACE}/.mvn/maven.config
                                echo "-Dwko.it.grafana.chart.version=\"${GRAFANA_CHART_VERSION}\""                           >> ${WORKSPACE}/.mvn/maven.config
                                echo "-Dwko.it.collect.logs.on.success=\"${COLLECT_LOGS_ON_SUCCESS}\""                       >> ${WORKSPACE}/.mvn/maven.config
                                echo "-Dwko.it.remoteconsole.version=\"${REMOTECONSOLE_VERSION}\""                           >> ${WORKSPACE}/.mvn/maven.config
				                echo "-Djdk.httpclient.allowRestrictedHeaders=\"host\""                                      >> ${WORKSPACE}/.mvn/maven.config

                                echo "${WORKSPACE}/.mvn/maven.config contents:"
                                cat "${WORKSPACE}/.mvn/maven.config"
                                cp "${WORKSPACE}/.mvn/maven.config" "${result_root}"
                            '''
                        }
                        withMaven(globalMavenSettingsConfig: 'wkt-maven-settings-xml', publisherStrategy: 'EXPLICIT') {
                            withCredentials([
                                usernamePassword(credentialsId: "${ocir_creds}", usernameVariable: 'OCIR_USER', passwordVariable: 'OCIR_PASS')
                            ]) {
                                sh '''
                                    export PATH=${runtime_path}
                                    export OKE_CLUSTER="true"
                                    export OKE_CLUSTER_PRIVATEIP="true"
                                    export OKD="false"
                                    export KUBECONFIG=${kubeconfig_file}
                                    export BASE_IMAGES_REPO_USERNAME="${OCIR_USER}"
                                    export BASE_IMAGES_REPO_PASSWORD="${OCIR_PASS}"
                                    export BASE_IMAGES_REPO_EMAIL="noreply@oracle.com"
                                    export TEST_IMAGES_REPO_USERNAME="${OCIR_USER}"
                                    export TEST_IMAGES_REPO_PASSWORD="${OCIR_PASS}"
                                    export TEST_IMAGES_REPO_EMAIL="noreply@oracle.com"
                                    
                                    if ! mvn -pl integration-tests -P ${MAVEN_PROFILE_NAME} verify 2>&1 | tee "${result_root}/oketest.log"; then
                                        echo "integration-tests failed"
                                        exit 1
                                    fi
                                '''
                            }
                        }
                    }
                    post {
                        always {
                            sh '''
                                export PATH="${WORKSPACE}/bin:${PATH}"
                                export KUBECONFIG=${kubeconfig_file}
                                export OCI_CLI_CONFIG_FILE=${WORKSPACE}/oci/config
                                export AVAILABILITY_DOMAIN=${availability_domain}
                                export OCI_CLI_PROFILE=${oci_profile}
                                mkdir -m777 -p ${result_root}/kubelogs
                                mkdir -m777 -p "${WORKSPACE}/logdir/${BUILD_TAG}/wl_k8s_test_results"
                                sudo mv -f ${result_root}/* "${WORKSPACE}/logdir/${BUILD_TAG}/wl_k8s_test_results"
                                ${WORKSPACE}/terraform/oke.delete.sh ${OCI_PROP_FILE} ${WORKSPACE}/terraform ${AVAILABILITY_DOMAIN}

                            '''

                            archiveArtifacts(artifacts:
                            "logdir/${BUILD_TAG}/wl_k8s_test_results/diagnostics/**/*,logdir/${BUILD_TAG}/wl_k8s_test_results/workdir/liftandshiftworkdir/**/*")
                            junit(testResults: 'integration-tests/target/failsafe-reports/*.xml', allowEmptyResults: true)
                        }
                    }
                }
            }
            post {
                always {
                    sh '''
                        export PATH="${WORKSPACE}/bin:${PATH}"
                        export KUBECONFIG=${kubeconfig_file}
                        export AVAILABILITY_DOMAIN=${availability_domain}

                        echo 'Remove old OKE cluster (if any)...'

                        if [ -f "$OCI_PROP_FILE" ] && [ -f "${WORKSPACE}/terraform/oke.delete.sh" ]; then
                            ${WORKSPACE}/terraform/oke.delete.sh ${OCI_PROP_FILE} ${WORKSPACE}/terraform ${AVAILABILITY_DOMAIN}
                        fi

                        if [ "${MAVEN_PROFILE_NAME}" = "oke-gate" ] && [ "${BRANCH}" = "main" ]; then
                            compname="wkt"
                            wkt_compartment_ocid=$(oci iam compartment list --compartment-id-in-subtree true --all | jq --arg compname "$compname" '.data[] | select(."name"==$compname)' | jq -r ."id")
                            sec_list_id=$(oci network security-list list --compartment-id="$wkt_compartment_ocid" --display-name=Security-List-wktiso1 | jq -r '.data[] | ."id"')
                            oci network security-list update --security-list-id="$sec_list_id" --ingress-security-rules='[{"description": "east west","icmp-options": null,"is-stateless": false,"protocol": "all","source": "10.196.0.0/16","source-type": "CIDR_BLOCK","tcp-options": null,"udp-options": null},{"description": null,"icmp-options": {"code": null,"type": 3},"is-stateless": false,"protocol": "1", "source": "10.196.0.0/16", "source-type": "CIDR_BLOCK","tcp-options": null, "udp-options": null}, {"description": null, "icmp-options": {"code": 4, "type": 3},"is-stateless": false, "protocol": "1", "source": "0.0.0.0/0","source-type": "CIDR_BLOCK", "tcp-options": null, "udp-options": null}, {"description": null, "icmp-options": null, "is-stateless": false, "protocol": "6","source": "0.0.0.0/0", "source-type": "CIDR_BLOCK", "tcp-options": null, "udp-options": null}, {"description": null, "icmp-options": null, "is-stateless": false,"protocol": "6", "source": "0.0.0.0/0", "source-type": "CIDR_BLOCK", "tcp-options": {"destination-port-range": {"max": 22, "min": 22}, "source-port-range": null}, "udp-options": null}]' --egress-security-rules='[{"description": null, "destination": "0.0.0.0/0", "destinationType": "CIDR_BLOCK", "icmpOptions": null, "isStateless": false, "protocol": "all", "tcpOptions": null, "udpOptions": null}, {"description": null, "destination": "oci-phx-objectstorage", "destinationType": "SERVICE_CIDR_BLOCK", "icmpOptions": null, "isStateless": false, "protocol": "6", "tcpOptions": null, "udpOptions": null}]' --force
                        fi
                        rm -f ${jenkins_home_directory}/.oci/config
                        rm -f ${jenkins_home_directory}/.oci/oci-signing-key.pem
                        rm -f ${WORKSPACE}/.ssh/*
                        rm -rf ${WORKSPACE}/.mvn

                    '''
                }
            }
        }
        stage ('Sync') {
             when {
                 anyOf {
                     branch 'main'
                     branch 'release/4.0'
                     branch 'release/3.4'
                 }
                 anyOf {
                     not { triggeredBy 'TimerTrigger' }
                     tag 'v*'
                 }
             }
             steps {
                 build job: "wkt-sync", parameters: [ string(name: 'REPOSITORY', value: 'weblogic-kubernetes-operator') ]
             }
        }
    }
}