HADOOP-17807. Use separate src dir for platform builds (#3210)

dev-support/Jenkinsfile

@@ -29,9 +29,6 @@ pipeline {
     }
 
     environment {
-        SOURCEDIR = 'src'
-        // will also need to change notification section below
-        PATCHDIR = 'out'
         YETUS='yetus'
         // Branch or tag name.  Yetus release tags are 'rel/X.Y.Z'
         YETUS_VERSION='f9ba0170a5787a5f4662d3769804fef0226a182f'
@@ -56,6 +53,35 @@ pipeline {
             }
         }
 
+        // Setup codebase so that each platform's build happens in its own exclusive copy of the
+        // codebase.
+        // Primarily because YETUS messes up the git branch information and affects the subsequent
+        // optional stages after the first one.
+        stage ('setup sources') {
+            steps {
+                dir("${WORKSPACE}/centos-8") {
+                    sh '''#!/usr/bin/env bash
+
+                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-8
+                    '''
+                }
+
+                dir("${WORKSPACE}/debian-10") {
+                    sh '''#!/usr/bin/env bash
+
+                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/debian-10
+                    '''
+                }
+
+                dir("${WORKSPACE}/ubuntu-focal") {
+                    sh '''#!/usr/bin/env bash
+
+                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/ubuntu-focal
+                    '''
+                }
+            }
+        }
+
         // This is an optional stage which runs only when there's a change in
         // C++/C++ build/platform.
         // This stage serves as a means of cross platform validation, which is
@@ -63,6 +89,8 @@ pipeline {
         // break the Hadoop build on Centos 8.
         stage ('precommit-run Centos 8') {
             environment {
+                SOURCEDIR = "${WORKSPACE}/centos-8/src"
+                PATCHDIR = "${WORKSPACE}/centos-8/out"
                 DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_8"
                 IS_OPTIONAL = 1
             }
@@ -78,10 +106,32 @@ pipeline {
                         sh '''#!/usr/bin/env bash
 
                         chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-                        "${SOURCEDIR}/dev-support/jenkins.sh"
+                        "${SOURCEDIR}/dev-support/jenkins.sh" run_ci
                         '''
                 }
             }
+
+            post {
+                // Since this is an optional platform, we want to copy the artifacts
+                // and archive it only if the build fails, to help with debugging.
+                failure {
+                    sh '''#!/usr/bin/env bash
+
+                    cp -Rp "${WORKSPACE}/centos-8/out" "${WORKSPACE}"
+                    '''
+                    archiveArtifacts "out/**"
+                }
+
+                cleanup() {
+                    script {
+                        sh '''#!/usr/bin/env bash
+
+                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
+                        "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
+                        '''
+                    }
+                }
+            }
         }
 
         // This is an optional stage which runs only when there's a change in
@@ -91,6 +141,8 @@ pipeline {
         // break the Hadoop build on Debian 10.
         stage ('precommit-run Debian 10') {
             environment {
+                SOURCEDIR = "${WORKSPACE}/debian-10/src"
+                PATCHDIR = "${WORKSPACE}/debian-10/out"
                 DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_debian_10"
                 IS_OPTIONAL = 1
             }
@@ -106,16 +158,40 @@ pipeline {
                         sh '''#!/usr/bin/env bash
 
                         chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-                        "${SOURCEDIR}/dev-support/jenkins.sh"
+                        "${SOURCEDIR}/dev-support/jenkins.sh" run_ci
                         '''
                 }
             }
+
+            post {
+                // Since this is an optional platform, we want to copy the artifacts
+                // and archive it only if the build fails, to help with debugging.
+                failure {
+                    sh '''#!/usr/bin/env bash
+
+                    cp -Rp "${WORKSPACE}/debian-10/out" "${WORKSPACE}"
+                    '''
+                    archiveArtifacts "out/**"
+                }
+
+                cleanup() {
+                    script {
+                        sh '''#!/usr/bin/env bash
+
+                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
+                        "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
+                        '''
+                    }
+                }
+            }
         }
 
         // We want to use Ubuntu Focal as our main CI and thus, this stage
         // isn't optional (runs for all the PRs).
         stage ('precommit-run Ubuntu focal') {
             environment {
+                SOURCEDIR = "${WORKSPACE}/ubuntu-focal/src"
+                PATCHDIR = "${WORKSPACE}/ubuntu-focal/out"
                 DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile"
                 IS_OPTIONAL = 0
             }
@@ -131,69 +207,76 @@ pipeline {
                         sh '''#!/usr/bin/env bash
 
                         chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
-                        "${SOURCEDIR}/dev-support/jenkins.sh"
+                        "${SOURCEDIR}/dev-support/jenkins.sh" run_ci
                         '''
                 }
             }
-        }
 
-    }
+            post {
+                always {
+                    script {
+                        // Publish status if it was missed (YETUS-1059)
+                        withCredentials(
+                            [usernamePassword(credentialsId: '683f5dcf-5552-4b28-9fb1-6a6b77cf53dd',
+                                              passwordVariable: 'GITHUB_TOKEN',
+                                              usernameVariable: 'GITHUB_USER')]) {
+                            sh '''#!/usr/bin/env bash
 
-    post {
-        always {
-          script {
-            // Publish status if it was missed (YETUS-1059)
-            withCredentials(
-                [usernamePassword(credentialsId: '683f5dcf-5552-4b28-9fb1-6a6b77cf53dd',
-                                  passwordVariable: 'GITHUB_TOKEN',
-                                  usernameVariable: 'GITHUB_USER')]) {
-                sh '''#!/usr/bin/env bash
-                    YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")
-                    YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
-                    TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/github-status-recovery.sh"
-                    /usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" ${EXTRA_ARGS} || true
-                    '''
-            }
+                            # Copy the artifacts of Ubuntu focal build to workspace
+                            cp -Rp "${WORKSPACE}/ubuntu-focal/out" "${WORKSPACE}"
+
+                            # Send Github status
+                            chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
+                            "${SOURCEDIR}/dev-support/jenkins.sh" github_status_recovery
+                            '''
+                        }
 
-            // Yetus output
-            archiveArtifacts "${env.PATCHDIR}/**"
-            // Publish the HTML report so that it can be looked at
-            // Has to be relative to WORKSPACE.
-            publishHTML (target: [
-                          allowMissing: true,
-                          keepAll: true,
-                          alwaysLinkToLastBuild: true,
-                          // Has to be relative to WORKSPACE
-                          reportDir: "${env.PATCHDIR}",
-                          reportFiles: 'report.html',
-                          reportName: 'Yetus Report'
-            ])
-            // Publish JUnit results
-            try {
-                junit "${env.SOURCEDIR}/**/target/surefire-reports/*.xml"
-            } catch(e) {
-                echo 'junit processing: ' + e.toString()
+                        // YETUS output
+                        archiveArtifacts "out/**"
+
+                        // Publish the HTML report so that it can be looked at
+                        // Has to be relative to WORKSPACE.
+                        publishHTML (target: [
+                                      allowMissing: true,
+                                      keepAll: true,
+                                      alwaysLinkToLastBuild: true,
+                                      // Has to be relative to WORKSPACE
+                                      reportDir: "out",
+                                      reportFiles: 'report.html',
+                                      reportName: 'Yetus Report'
+                        ])
+
+                        // Publish JUnit results
+                        try {
+                            junit "${SOURCEDIR}/**/target/surefire-reports/*.xml"
+                        } catch(e) {
+                            echo 'junit processing: ' + e.toString()
+                        }
+                    }
+                }
+
+                cleanup() {
+                    script {
+                        sh '''#!/usr/bin/env bash
+
+                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
+                        "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
+                        '''
+                    }
+                }
             }
-          }
         }
+    }
 
+    post {
         // Jenkins pipeline jobs fill slaves on PRs without this :(
         cleanup() {
             script {
-                sh '''
-                    # See YETUS-764
-                    if [ -f "${WORKSPACE}/${PATCHDIR}/pidfile.txt" ]; then
-                      echo "test-patch process appears to still be running: killing"
-                      kill `cat "${WORKSPACE}/${PATCHDIR}/pidfile.txt"` || true
-                      sleep 10
-                    fi
-                    if [ -f "${WORKSPACE}/${PATCHDIR}/cidfile.txt" ]; then
-                      echo "test-patch container appears to still be running: killing"
-                      docker kill `cat "${WORKSPACE}/${PATCHDIR}/cidfile.txt"` || true
-                    fi
-                    # See HADOOP-13951
-                    chmod -R u+rxw "${WORKSPACE}"
-                    '''
+                sh '''#!/usr/bin/env bash
+
+                # See HADOOP-13951
+                chmod -R u+rxw "${WORKSPACE}"
+                '''
                 deleteDir()
             }
         }

dev-support/jenkins.sh

@@ -115,10 +115,10 @@ function run_ci() {
   TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/test-patch.sh"
 
   # this must be clean for every run
-  if [[ -d "${WORKSPACE}/${PATCHDIR}" ]]; then
-    rm -rf "${WORKSPACE:?}/${PATCHDIR}"
+  if [[ -d "${PATCHDIR}" ]]; then
+    rm -rf "${PATCHDIR:?}"
   fi
-  mkdir -p "${WORKSPACE}/${PATCHDIR}"
+  mkdir -p "${PATCHDIR}"
 
   # if given a JIRA issue, process it. If CHANGE_URL is set
   # (e.g., Github Branch Source plugin), process it.
@@ -128,23 +128,23 @@ function run_ci() {
   if [[ -n "${JIRA_ISSUE_KEY}" ]]; then
     YETUS_ARGS+=("${JIRA_ISSUE_KEY}")
   elif [[ -z "${CHANGE_URL}" ]]; then
-    echo "Full build skipped" >"${WORKSPACE}/${PATCHDIR}/report.html"
+    echo "Full build skipped" >"${PATCHDIR}/report.html"
     exit 0
   fi
 
-  YETUS_ARGS+=("--patch-dir=${WORKSPACE}/${PATCHDIR}")
+  YETUS_ARGS+=("--patch-dir=${PATCHDIR}")
 
   # where the source is located
-  YETUS_ARGS+=("--basedir=${WORKSPACE}/${SOURCEDIR}")
+  YETUS_ARGS+=("--basedir=${SOURCEDIR}")
 
   # our project defaults come from a personality file
   YETUS_ARGS+=("--project=hadoop")
-  YETUS_ARGS+=("--personality=${WORKSPACE}/${SOURCEDIR}/dev-support/bin/hadoop.sh")
+  YETUS_ARGS+=("--personality=${SOURCEDIR}/dev-support/bin/hadoop.sh")
 
   # lots of different output formats
-  YETUS_ARGS+=("--brief-report-file=${WORKSPACE}/${PATCHDIR}/brief.txt")
-  YETUS_ARGS+=("--console-report-file=${WORKSPACE}/${PATCHDIR}/console.txt")
-  YETUS_ARGS+=("--html-report-file=${WORKSPACE}/${PATCHDIR}/report.html")
+  YETUS_ARGS+=("--brief-report-file=${PATCHDIR}/brief.txt")
+  YETUS_ARGS+=("--console-report-file=${PATCHDIR}/console.txt")
+  YETUS_ARGS+=("--html-report-file=${PATCHDIR}/report.html")
 
   # enable writing back to Github
   YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")
@@ -206,7 +206,49 @@ function run_ci() {
   "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
 }
 
-# Check if the CI needs to be run, if so, do so :)
-if check_ci_run; then
-  run_ci
+## @description  Cleans up the processes started by YETUS
+function cleanup_ci_proc() {
+  # See YETUS-764
+  if [ -f "${PATCHDIR}/pidfile.txt" ]; then
+    echo "test-patch process appears to still be running: killing"
+    kill "$(cat "${PATCHDIR}/pidfile.txt")" || true
+    sleep 10
+  fi
+  if [ -f "${PATCHDIR}/cidfile.txt" ]; then
+    echo "test-patch container appears to still be running: killing"
+    docker kill "$(cat "${PATCHDIR}/cidfile.txt")" || true
+  fi
+}
+
+## @description  Invokes github_status_recovery in YETUS's precommit
+function github_status_recovery() {
+  YETUS_ARGS+=("--github-token=${GITHUB_TOKEN}")
+  YETUS_ARGS+=("--patch-dir=${PATCHDIR}")
+  TESTPATCHBIN="${WORKSPACE}/${YETUS}/precommit/src/main/shell/github-status-recovery.sh"
+  /usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}" "${EXTRA_ARGS}" || true
+}
+
+if [ -z "$1" ]; then
+  echo "Must specify an argument for jenkins.sh"
+  echo "run_ci                  - Runs the CI based on platform image as defined by DOCKERFILE"
+  echo "cleanup_ci_proc         - Cleans up the processes spawned for running the CI"
+  echo "github_status_recovery  - Sends Github status (refer to YETUS precommit for more details)"
+  exit 1
+fi
+
+# Process arguments to jenkins.sh
+if [ "$1" == "run_ci" ]; then
+  # Check if the CI needs to be run, if so, do so :)
+  if check_ci_run; then
+    run_ci
+  else
+    echo "No C++ file/C++ build/platform changes found, will not run CI"
+  fi
+elif [ "$1" == "cleanup_ci_proc" ]; then
+  cleanup_ci_proc
+elif [ "$1" == "github_status_recovery" ]; then
+  github_status_recovery
+else
+  echo "Don't know how to process $1"
+  exit 1
 fi