MapR [DFDEVOPS-3022] Add an option to easily modify build images per component (apache#1153)

Co-authored-by: Anatoliy Shevchuk <[email protected]>

Author: idzikovskyi

devops/Jenkinsfile

@@ -18,7 +18,7 @@ env.ARTIFACTORY_PATH=''
 env.ID=''
 env.PROJECT=''
 
-env.DOCKER_REGISTRY="${DOCKER_REGISTRY}"
+IMAGE_NAMES_MAP=[:]
 
 pipeline {
   agent none
@@ -45,31 +45,30 @@ pipeline {
         }
       }
     }
-
     stage("Build & Sign & Upload to Artifactory") {
       agent none
       steps {
         script {
           buildInParallel=[:]
           YAML_CONFIG.build.each {
             buildInParallel["Build ${it.name}"] = { ->
-
               node (it.hostmachine_jenkins_label) {
                 checkout scm
-                build(it)
+                validateBuildImage(it)
+                echo "Using image: ${IMAGE_NAMES_MAP[it.name]}"
+                buildStage(it)
                 if(it.rpmSign == true)
                   signing.rpmSign("dist")
-                getVersionAndUploadToArtifactory()
+                getVersionAndUploadToArtifactory(it)
               }
-
             }
-          }
+         }
           parallel buildInParallel
         }
       }
     }
 
-    stage ("Whitesource Scan") {
+    stage ("OWASP Scan") {
       agent {label "redhat8"}
       when {
         equals (actual: YAML_CONFIG.whitesourceScan, expected: true)
@@ -78,18 +77,6 @@ pipeline {
         script {
           checkout scm
           ws.dockerScan()
-        /*Get token of product. We have to communicate with WS API only using 64-digits project's token.
-          We are sending request to API and get ALL "project+token" pairs. Then parse this information and find token for current project*/
-          productToken = ws.getWSProductToken()
-
-          //Get reports (Excel and json) from WS site and save these reports to files. Once for each format
-          ws.getWSReports(productToken)
-
-          //Move latest reports to the "old" folder in Artifactory
-          ws.moveOldWsReports(ARTIFACTORY_SERVER)
-
-          //Upload reports (Excel and json) to the Artifactory (to the latest folder)
-          ws.uploadToArtifactory(ARTIFACTORY_SERVER)
         }
       }
     }
@@ -124,42 +111,14 @@ pipeline {
   }
 }
 
-def build(yamlCurrentBuildItem) {
-  def image = yamlCurrentBuildItem.image
-  def imageContext = yamlCurrentBuildItem.imageContext
-
-  if (image) {
-    image = image.replace("_DOCKER_REGISTRY_", DOCKER_REGISTRY)
-  }
-
-  if (imageContext) {
-    if (image) {
-      error "You can not specify 'image' and 'imageContext' options in build-config.yaml"
-    }
-    def imageTagScript = yamlCurrentBuildItem.imageTagScript
-    image = sh script: imageTagScript, returnStdout: true
-    image = image.trim()
-    def imageCheckStatus = sh script: "docker inspect -f . '${image}'", returnStatus: true
-    if (imageCheckStatus != 0) {
-      imageCheckStatus = sh script: "docker pull '${image}'", returnStatus: true
-    }
-    if (imageCheckStatus != 0) {
-      sh """
-        docker build \
-          --build-arg DOCKER_REGISTRY='${DOCKER_REGISTRY}' \
-          -t '${image}' \
-          '${imageContext}'
-        docker push '${image}'
-      """
-    }
-  }
-
+def buildStage(yamlCurrentBuildItem) {
   def buildCommand = yamlCurrentBuildItem.basicBuildCommands.replace("_BUILD_ENV_VARS_","TIMESTAMP=${ID} DO_DEPLOY=${params.DO_DEPLOY} ${params.BUILD_ENV_VARS}")
 
   configFileProvider([configFile(fileId: 'maven_settings_spark_deploy_mvn381_above', variable: 'mvn_settings')]) {
     sh "cat ${env.mvn_settings} > settings.xml"
   }
-  docker.image(image).inside (
+
+  docker.image(IMAGE_NAMES_MAP[yamlCurrentBuildItem.name]).inside (
   "-e MAPR_MIRROR=${MAPR_MIRROR} \
   -e MAPR_CENTRAL=${MAPR_CENTRAL} \
   -e MAVEN_CENTRAL=${MAVEN_CENTRAL} \
@@ -185,7 +144,25 @@ def build(yamlCurrentBuildItem) {
   }
 }
 
-def getVersionAndUploadToArtifactory() {
-  def osVersionContainerRaw = sh script: 'cat os_release_version.txt', returnStdout:true
-  artifactory.uploadToArtifactoryWithOSVersions(ARTIFACTORY_SERVER, osVersionContainerRaw)
+def validateBuildImage(yamlCurrentBuildItem) {
+    def imageStatic = yamlCurrentBuildItem.image
+    def imageCustom = yamlCurrentBuildItem.imageCustom
+    def imageName = ""
+    if (imageStatic) {
+      imageName = imageStatic.replace("_DOCKER_REGISTRY_", DOCKER_REGISTRY)
+    }
+    if (imageCustom) {
+      if (imageStatic) {
+        error "You can not specify 'image' and 'imageCustom' options in build-config.yaml"
+      }
+      imageName = sh(script: imageCustom, returnStdout: true).trim()
+    }
+    IMAGE_NAMES_MAP[yamlCurrentBuildItem.name] = imageName
+}
+
+def getVersionAndUploadToArtifactory(yamlCurrentBuildItem) {
+  stage("Get Version And Upload to Artifactory stage:${yamlCurrentBuildItem.name}") {
+    def osVersionContainerRaw = sh script: 'cat os_release_version.txt', returnStdout:true
+    artifactory.uploadToArtifactoryWithOSVersions(ARTIFACTORY_SERVER, osVersionContainerRaw)
+  }
 }

devops/build-config.yaml

@@ -4,13 +4,11 @@ initialReleaseTag: '3.3.3.0-eep-921' # Use only for release build
 whitesourceScan: true
 build:
   - name: ubuntu
-    imageContext: devops/buildimages/ubuntu/
-    imageTagScript: devops/buildimages/ubuntu/tag.sh
+    imageCustom: devops/buildimages/ubuntu/image.sh
     basicBuildCommands: "_BUILD_ENV_VARS_ devops/scripts/build.sh"
     hostmachine_jenkins_label: "ubuntu16"
   - name: centos
-    imageContext: devops/buildimages/centos/
-    imageTagScript: devops/buildimages/centos/tag.sh
+    imageCustom: devops/buildimages/centos/image.sh
     basicBuildCommands: "_BUILD_ENV_VARS_ devops/scripts/build.sh"
     hostmachine_jenkins_label: "redhat8"
     rpmSign: true

devops/buildimages/centos/Dockerfile

@@ -1,5 +1,5 @@
-ARG DOCKER_REGISTRY
-FROM ${DOCKER_REGISTRY}/centos8-java11-gcc8
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
 
 RUN sed -i 's/^mirrorlist/#mirrorlist/' /etc/yum.repos.d/CentOS-* && \
     sed -i 's/^#baseurl/baseurl/' /etc/yum.repos.d/CentOS-* && \

devops/buildimages/centos/image.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+BASE_IMAGE=${DOCKER_REGISTRY}/centos8-java11-gcc8
+BASE_TAG=latest
+
+BASEDIR=$(dirname $0)
+DOCKERFILE_CHECKSUM=$(md5sum "${BASEDIR}/Dockerfile" | head -c 7)
+
+IMAGE_NAME_FOR_BUILD=${BASE_IMAGE}-custom-${REPOSITORY_NAME}:${BASE_TAG}-${DOCKERFILE_CHECKSUM}
+
+ensure_image() {
+    docker image pull -q ${IMAGE_NAME_FOR_BUILD}
+    retVal=$?
+    if [ "$retVal" -ne 0 ]; then
+        docker build -q ${BASEDIR} -f ${BASEDIR}/Dockerfile -t ${IMAGE_NAME_FOR_BUILD} --build-arg BASE_IMAGE=${BASE_IMAGE}:${BASE_TAG}
+        docker push -q ${IMAGE_NAME_FOR_BUILD}
+    fi
+}
+
+ensure_image >&2
+
+echo ${IMAGE_NAME_FOR_BUILD}

devops/buildimages/centos/tag.sh

@@ -1,5 +1,5 @@
-ARG DOCKER_REGISTRY
-FROM ${DOCKER_REGISTRY}/ubuntu16-java11-gcc7
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
 
 RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 51716619E084DAB9 && \
     add-apt-repository 'deb [trusted=yes] https://cloud.r-project.org/bin/linux/ubuntu xenial-cran35/' && \

devops/buildimages/ubuntu/Dockerfile

@@ -0,0 +1,20 @@
+BASE_IMAGE=${DOCKER_REGISTRY}/ubuntu16-java11-gcc7
+BASE_TAG=latest
+
+BASEDIR=$(dirname $0)
+DOCKERFILE_CHECKSUM=$(md5sum "${BASEDIR}/Dockerfile" | head -c 7)
+
+IMAGE_NAME_FOR_BUILD=${BASE_IMAGE}-custom-${REPOSITORY_NAME}:${BASE_TAG}-${DOCKERFILE_CHECKSUM}
+
+ensure_image() {
+    docker image pull -q ${IMAGE_NAME_FOR_BUILD}
+    retVal=$?
+    if [ "$retVal" -ne 0 ]; then
+        docker build -q ${BASEDIR} -f ${BASEDIR}/Dockerfile -t ${IMAGE_NAME_FOR_BUILD} --build-arg BASE_IMAGE=${BASE_IMAGE}:${BASE_TAG}
+        docker push -q ${IMAGE_NAME_FOR_BUILD}
+    fi
+}
+
+ensure_image >&2
+
+echo ${IMAGE_NAME_FOR_BUILD}