HADOOP-17815. Run CI for Centos 7 (#3231)

dev-support/Jenkinsfile

@@ -59,6 +59,13 @@ pipeline {
         // optional stages after the first one.
         stage ('setup sources') {
             steps {
+                dir("${WORKSPACE}/centos-7") {
+                    sh '''#!/usr/bin/env bash
+
+                    cp -Rp ${WORKSPACE}/src ${WORKSPACE}/centos-7
+                    '''
+                }
+
                 dir("${WORKSPACE}/centos-8") {
                     sh '''#!/usr/bin/env bash
 
@@ -82,6 +89,58 @@ pipeline {
             }
         }
 
+        // This is an optional stage which runs only when there's a change in
+        // C++/C++ build/platform.
+        // This stage serves as a means of cross platform validation, which is
+        // really needed to ensure that any C++ related/platform change doesn't
+        // break the Hadoop build on Centos 7.
+        stage ('precommit-run Centos 7') {
+            environment {
+                SOURCEDIR = "${WORKSPACE}/centos-7/src"
+                PATCHDIR = "${WORKSPACE}/centos-7/out"
+                DOCKERFILE = "${SOURCEDIR}/dev-support/docker/Dockerfile_centos_7"
+                IS_OPTIONAL = 1
+            }
+
+            steps {
+                withCredentials(
+                    [usernamePassword(credentialsId: 'apache-hadoop-at-github.com',
+                                  passwordVariable: 'GITHUB_TOKEN',
+                                  usernameVariable: 'GITHUB_USER'),
+                    usernamePassword(credentialsId: 'hadoopqa-at-asf-jira',
+                                        passwordVariable: 'JIRA_PASSWORD',
+                                        usernameVariable: 'JIRA_USER')]) {
+                        sh '''#!/usr/bin/env bash
+
+                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
+                        "${SOURCEDIR}/dev-support/jenkins.sh" run_ci
+                        '''
+                }
+            }
+
+            post {
+                // Since this is an optional platform, we want to copy the artifacts
+                // and archive it only if the build fails, to help with debugging.
+                failure {
+                    sh '''#!/usr/bin/env bash
+
+                    cp -Rp "${WORKSPACE}/centos-7/out" "${WORKSPACE}"
+                    '''
+                    archiveArtifacts "out/**"
+                }
+
+                cleanup() {
+                    script {
+                        sh '''#!/usr/bin/env bash
+
+                        chmod u+x "${SOURCEDIR}/dev-support/jenkins.sh"
+                        "${SOURCEDIR}/dev-support/jenkins.sh" cleanup_ci_proc
+                        '''
+                    }
+                }
+            }
+        }
+
         // This is an optional stage which runs only when there's a change in
         // C++/C++ build/platform.
         // This stage serves as a means of cross platform validation, which is

dev-support/docker/Dockerfile_centos_7

@@ -35,6 +35,7 @@ RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
 ######
 # hadolint ignore=DL3008,SC2046
 RUN yum update -y \
+    && yum groupinstall -y "Development Tools" \
     && yum install -y \
         centos-release-scl \
         python3 \
@@ -44,6 +45,21 @@ RUN yum update -y \
 RUN echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc
 SHELL ["/bin/bash", "--login", "-c"]
 
+######
+# Set the environment variables needed for CMake
+# to find and use GCC 9 for compilation
+######
+ENV GCC_HOME "/opt/rh/devtoolset-9"
+ENV CC "${GCC_HOME}/root/usr/bin/gcc"
+ENV CXX "${GCC_HOME}/root/usr/bin/g++"
+ENV SHLVL 1
+ENV LD_LIBRARY_PATH "${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64"
+ENV PCP_DIR "${GCC_HOME}/root"
+ENV MANPATH "${GCC_HOME}/root/usr/share/man:"
+ENV PATH "${GCC_HOME}/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+ENV PKG_CONFIG_PATH "${GCC_HOME}/root/usr/lib64/pkgconfig"
+ENV INFOPATH "${GCC_HOME}/root/usr/share/info"
+
 # TODO: Set locale
 
 ######
@@ -65,7 +81,6 @@ ENV SPOTBUGS_HOME /opt/spotbugs
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"
 
-
 ######
 # Install packages
 ######
@@ -76,4 +91,6 @@ RUN pkg-resolver/install-yasm.sh centos:7
 RUN pkg-resolver/install-protobuf.sh centos:7
 RUN pkg-resolver/install-boost.sh centos:7
 RUN pkg-resolver/install-spotbugs.sh centos:7
+RUN pkg-resolver/install-nodejs.sh centos:7
+RUN pkg-resolver/install-git.sh centos:7
 RUN pkg-resolver/install-common-pkgs.sh

dev-support/docker/pkg-resolver/install-git.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ]; then
+  echo "ERROR: Need at least 1 argument, $# were provided"
+  exit 1
+fi
+
+pkg-resolver/check_platform.py "$1"
+if [ $? -eq 1 ]; then
+  echo "ERROR: Unsupported platform $1"
+  exit 1
+fi
+
+default_version="2.9.5"
+version_to_install=$default_version
+if [ -n "$2" ]; then
+  version_to_install="$2"
+fi
+
+if [ "$version_to_install" != "2.9.5" ]; then
+  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
+  version_to_install=$default_version
+fi
+
+if [ "$version_to_install" == "2.9.5" ]; then
+  # hadolint ignore=DL3003
+  mkdir -p /tmp/git /opt/git &&
+    curl -L -s -S https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.9.5.tar.gz >/tmp/git/git-2.9.5.tar.gz &&
+    tar xzf /tmp/git/git-2.9.5.tar.gz --strip-components 1 -C /opt/git &&
+    cd /opt/git || exit &&
+    make configure &&
+    ./configure --prefix=/usr/local &&
+    make "-j$(nproc)" &&
+    make install &&
+    cd /root || exit
+else
+  echo "ERROR: Don't know how to install version $version_to_install"
+  exit 1
+fi

dev-support/docker/pkg-resolver/install-nodejs.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ]; then
+  echo "ERROR: Need at least 1 argument, $# were provided"
+  exit 1
+fi
+
+pkg-resolver/check_platform.py "$1"
+if [ $? -eq 1 ]; then
+  echo "ERROR: Unsupported platform $1"
+  exit 1
+fi
+
+default_version="14.16.1"
+version_to_install=$default_version
+if [ -n "$2" ]; then
+  version_to_install="$2"
+fi
+
+if [ "$version_to_install" != "14.16.1" ]; then
+  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
+  version_to_install=$default_version
+fi
+
+if [ "$version_to_install" == "14.16.1" ]; then
+  # hadolint ignore=DL3003
+  mkdir -p /tmp/node &&
+    curl -L -s -S https://nodejs.org/dist/v14.16.1/node-v14.16.1.tar.gz -o /tmp/node-v14.16.1.tar.gz &&
+    tar xzf /tmp/node-v14.16.1.tar.gz --strip-components 1 -C /tmp/node &&
+    cd /tmp/node || exit &&
+    ./configure &&
+    make "-j$(nproc)" &&
+    make install &&
+    cd /root || exit
+else
+  echo "ERROR: Don't know how to install version $version_to_install"
+  exit 1
+fi

dev-support/docker/pkg-resolver/packages.json

@@ -18,6 +18,9 @@
     "centos:7": "automake",
     "centos:8": "automake"
   },
+  "autoconf": {
+    "centos:7": "autoconf"
+  },
   "bats": {
     "debian:10": "bats",
     "ubuntu:focal": "bats",
@@ -136,11 +139,13 @@
       "devtoolset-9"
     ]
   },
+  "gettext": {
+    "centos:7": "gettext-devel"
+  },
   "git": {
     "debian:10": "git",
     "ubuntu:focal": "git",
     "ubuntu:focal::arch64": "git",
-    "centos:7": "git",
     "centos:8": "git"
   },
   "gnupg-agent": {
@@ -172,6 +177,12 @@
     "centos:7": "openssl-devel",
     "centos:8": "openssl-devel"
   },
+  "perl": {
+    "centos:7": [
+      "perl-CPAN",
+      "perl-devel"
+    ]
+  },
   "protocol-buffers": {
     "debian:10": [
       "libprotobuf-dev",