HADOOP-17880. Build Hadoop on Centos 7 (#3535)

dev-support/docker/Dockerfile_centos_7

@@ -0,0 +1,97 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Dockerfile for installing the necessary dependencies for building Hadoop.
+# See BUILDING.txt.
+
+FROM centos:7
+
+WORKDIR /root
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+######
+# Platform package dependency resolver
+######
+COPY pkg-resolver pkg-resolver
+RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
+    && chmod a+r pkg-resolver/*.json
+
+######
+# Install packages from yum
+######
+# because python-pip not found at centos-release-scl repo, we need to install epel-release
+# hadolint ignore=DL3008,SC2046
+RUN yum update -y \
+    && yum groupinstall -y "Development Tools" \
+    && yum install -y \
+        centos-release-scl \
+        epel-release \
+        python \
+    && yum install -y $(pkg-resolver/resolve.py centos:7)
+
+# Set GCC 9 as the default C/C++ compiler
+RUN echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc
+SHELL ["/bin/bash", "--login", "-c"]
+
+######
+# Set the environment variables needed for CMake
+# to find and use GCC 9 for compilation
+######
+ENV GCC_HOME "/opt/rh/devtoolset-9"
+ENV CC "${GCC_HOME}/root/usr/bin/gcc"
+ENV CXX "${GCC_HOME}/root/usr/bin/g++"
+ENV SHLVL 1
+ENV LD_LIBRARY_PATH "${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:${GCC_HOME}/root/usr/lib64/dyninst:${GCC_HOME}/root/usr/lib/dyninst:${GCC_HOME}/root/usr/lib64:${GCC_HOME}/root/usr/lib:/usr/lib:/usr/lib64"
+ENV PCP_DIR "${GCC_HOME}/root"
+ENV MANPATH "${GCC_HOME}/root/usr/share/man:"
+ENV PATH "${GCC_HOME}/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+ENV PKG_CONFIG_PATH "${GCC_HOME}/root/usr/lib64/pkgconfig"
+ENV INFOPATH "${GCC_HOME}/root/usr/share/info"
+
+# TODO: Set locale
+
+######
+# Set env vars required to build Hadoop
+######
+ENV MAVEN_HOME /opt/maven
+ENV PATH "${PATH}:${MAVEN_HOME}/bin"
+# JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
+ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
+
+#######
+# Set env vars for SpotBugs
+#######
+ENV SPOTBUGS_HOME /opt/spotbugs
+
+#######
+# Set env vars for Google Protobuf
+#######
+ENV PROTOBUF_HOME /opt/protobuf
+ENV PATH "${PATH}:/opt/protobuf/bin"
+
+######
+# Install packages
+######
+RUN pkg-resolver/install-maven.sh centos:7
+RUN pkg-resolver/install-cmake.sh centos:7
+RUN pkg-resolver/install-zstandard.sh centos:7
+RUN pkg-resolver/install-yasm.sh centos:7
+RUN pkg-resolver/install-protobuf.sh centos:7
+RUN pkg-resolver/install-spotbugs.sh centos:7
+RUN pkg-resolver/install-nodejs.sh centos:7
+RUN pkg-resolver/install-git.sh centos:7
+RUN pkg-resolver/install-common-pkgs.sh

dev-support/docker/pkg-resolver/check_platform.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Checks whether the given platform is supported for building Apache Hadoop
+"""
+
+import json
+import os
+import sys
+
+
+def get_platforms():
+    """
+    :return: A list of the supported platforms managed by pkg-resolver.
+    """
+
+    with open('pkg-resolver/platforms.json', mode='rb') as platforms_file:
+        return json.loads(platforms_file.read().decode("UTF-8"))
+
+
+def is_supported_platform(platform):
+    """
+    :param platform: The name of the platform
+    :return: Whether the platform is supported
+    """
+    return platform in get_platforms()
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+        sys.stderr.write('ERROR: Expecting 1 argument, {} were provided{}'.format(
+            len(sys.argv) - 1, os.linesep))
+        sys.exit(1)
+
+    sys.exit(0 if is_supported_platform(sys.argv[1]) else 1)

dev-support/docker/pkg-resolver/install-cmake.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ]; then
+  echo "ERROR: Need at least 1 argument, $# were provided"
+  exit 1
+fi
+
+pkg-resolver/check_platform.py "$1"
+if [ $? -eq 1 ]; then
+  echo "ERROR: Unsupported platform $1"
+  exit 1
+fi
+
+default_version="3.19.0"
+version_to_install=$default_version
+if [ -n "$2" ]; then
+  version_to_install="$2"
+fi
+
+if [ "$version_to_install" != "3.19.0" ]; then
+  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
+  version_to_install=$default_version
+fi
+
+if [ "$version_to_install" == "3.19.0" ]; then
+  # hadolint ignore=DL3003
+  mkdir -p /tmp/cmake /opt/cmake &&
+    curl -L -s -S https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz -o /tmp/cmake/cmake-3.19.0.tar.gz &&
+    tar xzf /tmp/cmake/cmake-3.19.0.tar.gz --strip-components 1 -C /opt/cmake &&
+    cd /opt/cmake || exit && ./bootstrap &&
+    make "-j$(nproc)" &&
+    make install &&
+    cd /root || exit
+else
+  echo "ERROR: Don't know how to install version $version_to_install"
+  exit 1
+fi

dev-support/docker/pkg-resolver/install-common-pkgs.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+######
+# Install pylint and python-dateutil
+######
+pip2 install configparser==4.0.2 pylint==1.9.2 isort==4.3.21 python-dateutil==2.7.3

dev-support/docker/pkg-resolver/install-git.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ]; then
+  echo "ERROR: Need at least 1 argument, $# were provided"
+  exit 1
+fi
+
+pkg-resolver/check_platform.py "$1"
+if [ $? -eq 1 ]; then
+  echo "ERROR: Unsupported platform $1"
+  exit 1
+fi
+
+default_version="2.9.5"
+version_to_install=$default_version
+if [ -n "$2" ]; then
+  version_to_install="$2"
+fi
+
+if [ "$version_to_install" != "2.9.5" ]; then
+  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
+  version_to_install=$default_version
+fi
+
+if [ "$version_to_install" == "2.9.5" ]; then
+  # hadolint ignore=DL3003
+  mkdir -p /tmp/git /opt/git &&
+    curl -L -s -S https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.9.5.tar.gz >/tmp/git/git-2.9.5.tar.gz &&
+    tar xzf /tmp/git/git-2.9.5.tar.gz --strip-components 1 -C /opt/git &&
+    cd /opt/git || exit &&
+    make configure &&
+    ./configure --prefix=/usr/local &&
+    make "-j$(nproc)" &&
+    make install &&
+    cd /root || exit
+else
+  echo "ERROR: Don't know how to install version $version_to_install"
+  exit 1
+fi

dev-support/docker/pkg-resolver/install-maven.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ]; then
+  echo "ERROR: Need at least 1 argument, $# were provided"
+  exit 1
+fi
+
+pkg-resolver/check_platform.py "$1"
+if [ $? -eq 1 ]; then
+  echo "ERROR: Unsupported platform $1"
+  exit 1
+fi
+
+default_version="3.6.3"
+version_to_install=$default_version
+if [ -n "$2" ]; then
+  version_to_install="$2"
+fi
+
+if [ "$version_to_install" != "3.6.3" ]; then
+  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
+  version_to_install=$default_version
+fi
+
+if [ "$version_to_install" == "3.6.3" ]; then
+  mkdir -p /opt/maven /tmp/maven &&
+    curl -L -s -S https://mirrors.estointernet.in/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \
+      -o /tmp/maven/apache-maven-3.6.3-bin.tar.gz &&
+    tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven
+else
+  echo "ERROR: Don't know how to install version $version_to_install"
+  exit 1
+fi

dev-support/docker/pkg-resolver/install-nodejs.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ]; then
+  echo "ERROR: Need at least 1 argument, $# were provided"
+  exit 1
+fi
+
+pkg-resolver/check_platform.py "$1"
+if [ $? -eq 1 ]; then
+  echo "ERROR: Unsupported platform $1"
+  exit 1
+fi
+
+default_version="14.16.1"
+version_to_install=$default_version
+if [ -n "$2" ]; then
+  version_to_install="$2"
+fi
+
+if [ "$version_to_install" != "14.16.1" ]; then
+  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
+  version_to_install=$default_version
+fi
+
+if [ "$version_to_install" == "14.16.1" ]; then
+  # hadolint ignore=DL3003
+  mkdir -p /tmp/node &&
+    curl -L -s -S https://nodejs.org/dist/v14.16.1/node-v14.16.1.tar.gz -o /tmp/node-v14.16.1.tar.gz &&
+    tar xzf /tmp/node-v14.16.1.tar.gz --strip-components 1 -C /tmp/node &&
+    cd /tmp/node || exit &&
+    ./configure &&
+    make "-j$(nproc)" &&
+    make install &&
+    cd /root || exit
+else
+  echo "ERROR: Don't know how to install version $version_to_install"
+  exit 1
+fi

dev-support/docker/pkg-resolver/install-protobuf.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ]; then
+  echo "ERROR: Need at least 1 argument, $# were provided"
+  exit 1
+fi
+
+pkg-resolver/check_platform.py "$1"
+if [ $? -eq 1 ]; then
+  echo "ERROR: Unsupported platform $1"
+  exit 1
+fi
+
+default_version="2.5.0"
+version_to_install=$default_version
+if [ -n "$2" ]; then
+  version_to_install="$2"
+fi
+
+if [ "$version_to_install" != "2.5.0" ]; then
+  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
+  version_to_install=$default_version
+fi
+
+if [ "$version_to_install" == "2.5.0" ]; then
+  # hadolint ignore=DL3003
+  mkdir -p /opt/protobuf-src \
+    && curl -L -s -S \
+      https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz \
+      -o /opt/protobuf.tar.gz \
+    && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
+    && cd /opt/protobuf-src \
+    && ./configure --prefix=/opt/protobuf \
+    && make install \
+    && cd /root \
+    && rm -rf /opt/protobuf-src
+else
+  echo "ERROR: Don't know how to install version $version_to_install"
+  exit 1
+fi

dev-support/docker/pkg-resolver/install-spotbugs.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ]; then
+  echo "ERROR: Need at least 1 argument, $# were provided"
+  exit 1
+fi
+
+pkg-resolver/check_platform.py "$1"
+if [ $? -eq 1 ]; then
+  echo "ERROR: Unsupported platform $1"
+  exit 1
+fi
+
+default_version="4.2.2"
+version_to_install=$default_version
+if [ -n "$2" ]; then
+  version_to_install="$2"
+fi
+
+if [ "$version_to_install" != "4.2.2" ]; then
+  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
+  version_to_install=$default_version
+fi
+
+if [ "$version_to_install" == "4.2.2" ]; then
+  mkdir -p /opt/spotbugs &&
+    curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \
+      -o /opt/spotbugs.tgz &&
+    tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs &&
+    chmod +x /opt/spotbugs/bin/*
+else
+  echo "ERROR: Don't know how to install version $version_to_install"
+  exit 1
+fi

dev-support/docker/pkg-resolver/install-yasm.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ]; then
+  echo "ERROR: Need at least 1 argument, $# were provided"
+  exit 1
+fi
+
+pkg-resolver/check_platform.py "$1"
+if [ $? -eq 1 ]; then
+  echo "ERROR: Unsupported platform $1"
+  exit 1
+fi
+
+default_version="1.2.0-4"
+version_to_install=$default_version
+if [ -n "$2" ]; then
+  version_to_install="$2"
+fi
+
+if [ "$version_to_install" != "1.2.0-4" ]; then
+  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
+  version_to_install=$default_version
+fi
+
+if [ "$version_to_install" == "1.2.0-4" ]; then
+  mkdir -p /tmp/yasm &&
+    curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/y/yasm-1.2.0-4.el7.x86_64.rpm \
+      -o /tmp/yasm-1.2.0-4.el7.x86_64.rpm &&
+    rpm -Uvh /tmp/yasm-1.2.0-4.el7.x86_64.rpm
+else
+  echo "ERROR: Don't know how to install version $version_to_install"
+  exit 1
+fi

dev-support/docker/pkg-resolver/install-zstandard.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ]; then
+  echo "ERROR: Need at least 1 argument, $# were provided"
+  exit 1
+fi
+
+pkg-resolver/check_platform.py "$1"
+if [ $? -eq 1 ]; then
+  echo "ERROR: Unsupported platform $1"
+  exit 1
+fi
+
+default_version="1.4.9"
+version_to_install=$default_version
+if [ -n "$2" ]; then
+  version_to_install="$2"
+fi
+
+if [ "$version_to_install" != "1.4.9" ]; then
+  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
+  version_to_install=$default_version
+fi
+
+if [ "$version_to_install" == "1.4.9" ]; then
+  # hadolint ignore=DL3003
+  mkdir -p /opt/zstd /tmp/zstd &&
+    curl -L -s -S https://github.com/facebook/zstd/archive/refs/tags/v1.4.9.tar.gz -o /tmp/zstd/v1.4.9.tar.gz &&
+    tar xzf /tmp/zstd/v1.4.9.tar.gz --strip-components 1 -C /opt/zstd &&
+    cd /opt/zstd || exit &&
+    make "-j$(nproc)" &&
+    make install &&
+    cd /root || exit
+else
+  echo "ERROR: Don't know how to install version $version_to_install"
+  exit 1
+fi

dev-support/docker/pkg-resolver/packages.json

@@ -0,0 +1,359 @@
+{
+  "ant": {
+    "debian:10": "ant",
+    "ubuntu:focal": "ant",
+    "ubuntu:focal::arch64": "ant",
+    "centos:7": "ant",
+    "centos:8": "ant"
+  },
+  "apt-utils": {
+    "debian:10": "apt-utils",
+    "ubuntu:focal": "apt-utils",
+    "ubuntu:focal::arch64": "apt-utils"
+  },
+  "automake": {
+    "debian:10": "automake",
+    "ubuntu:focal": "automake",
+    "ubuntu:focal::arch64": "automake",
+    "centos:7": "automake",
+    "centos:8": "automake"
+  },
+  "autoconf": {
+    "centos:7": "autoconf"
+  },
+  "bats": {
+    "debian:10": "bats",
+    "ubuntu:focal": "bats",
+    "ubuntu:focal::arch64": "bats"
+  },
+  "build-essential": {
+    "debian:10": "build-essential",
+    "ubuntu:focal": "build-essential",
+    "ubuntu:focal::arch64": "build-essential",
+    "centos:7": "build-essential"
+  },
+  "bzip2": {
+    "debian:10": [
+      "bzip2",
+      "libbz2-dev"
+    ],
+    "ubuntu:focal": [
+      "bzip2",
+      "libbz2-dev"
+    ],
+    "ubuntu:focal::arch64": [
+      "bzip2",
+      "libbz2-dev"
+    ],
+    "centos:7": [
+      "bzip2",
+      "bzip2-devel"
+    ],
+    "centos:8": [
+      "bzip2",
+      "bzip2-devel"
+    ]
+  },
+  "clang": {
+    "debian:10": "clang",
+    "ubuntu:focal": "clang",
+    "ubuntu:focal::arch64": "clang",
+    "centos:7": "clang",
+    "centos:8": "clang"
+  },
+  "cmake": {
+    "debian:10": "cmake",
+    "ubuntu:focal": "cmake",
+    "ubuntu:focal::arch64": "cmake"
+  },
+  "curl": {
+    "debian:10": [
+      "curl",
+      "libcurl4-openssl-dev"
+    ],
+    "ubuntu:focal": [
+      "curl",
+      "libcurl4-openssl-dev"
+    ],
+    "ubuntu:focal::arch64": [
+      "curl",
+      "libcurl4-openssl-dev"
+    ],
+    "centos:7": [
+      "curl",
+      "libcurl-devel"
+    ],
+    "centos:8": [
+      "curl",
+      "libcurl-devel"
+    ]
+  },
+  "doxygen": {
+    "debian:10": "doxygen",
+    "ubuntu:focal": "doxygen",
+    "ubuntu:focal::arch64": "doxygen",
+    "centos:7": "doxygen"
+  },
+  "dnf": {
+    "centos:8": "dnf"
+  },
+  "fuse": {
+    "debian:10": [
+      "fuse",
+      "libfuse-dev"
+    ],
+    "ubuntu:focal": [
+      "fuse",
+      "libfuse-dev"
+    ],
+    "ubuntu:focal::arch64": [
+      "fuse",
+      "libfuse-dev"
+    ],
+    "centos:7": [
+      "fuse",
+      "fuse-libs",
+      "fuse-devel"
+    ],
+    "centos:8": [
+      "fuse",
+      "fuse-libs",
+      "fuse-devel"
+    ]
+  },
+  "gcc": {
+    "debian:10": [
+      "gcc",
+      "g++"
+    ],
+    "ubuntu:focal": [
+      "gcc",
+      "g++"
+    ],
+    "ubuntu:focal::arch64": [
+      "gcc",
+      "g++"
+    ],
+    "centos:7": [
+      "devtoolset-9"
+    ]
+  },
+  "gettext": {
+    "centos:7": "gettext-devel"
+  },
+  "git": {
+    "debian:10": "git",
+    "ubuntu:focal": "git",
+    "ubuntu:focal::arch64": "git",
+    "centos:8": "git"
+  },
+  "gnupg-agent": {
+    "debian:10": "gnupg-agent",
+    "ubuntu:focal": "gnupg-agent",
+    "ubuntu:focal::arch64": "gnupg-agent"
+  },
+  "hugo": {
+    "debian:10": "hugo",
+    "ubuntu:focal": "hugo",
+    "ubuntu:focal::arch64": "hugo"
+  },
+  "libbcprov-java": {
+    "debian:10": "libbcprov-java",
+    "ubuntu:focal": "libbcprov-java",
+    "ubuntu:focal::arch64": "libbcprov-java"
+  },
+  "libtool": {
+    "debian:10": "libtool",
+    "ubuntu:focal": "libtool",
+    "ubuntu:focal::arch64": "libtool",
+    "centos:7": "libtool",
+    "centos:8": "libtool"
+  },
+  "openssl": {
+    "debian:10": "libssl-dev",
+    "ubuntu:focal": "libssl-dev",
+    "ubuntu:focal::arch64": "libssl-dev",
+    "centos:7": "openssl-devel",
+    "centos:8": "openssl-devel"
+  },
+  "perl": {
+    "centos:7": [
+      "perl-CPAN",
+      "perl-devel"
+    ]
+  },
+  "protocol-buffers": {
+    "debian:10": [
+      "libprotobuf-dev",
+      "libprotoc-dev"
+    ],
+    "ubuntu:focal": [
+      "libprotobuf-dev",
+      "libprotoc-dev"
+    ],
+    "ubuntu:focal::arch64": [
+      "libprotobuf-dev",
+      "libprotoc-dev"
+    ]
+  },
+  "sasl": {
+    "debian:10": "libsasl2-dev",
+    "ubuntu:focal": "libsasl2-dev",
+    "ubuntu:focal::arch64": "libsasl2-dev",
+    "centos:7": "cyrus-sasl-devel",
+    "centos:8": "cyrus-sasl-devel"
+  },
+  "snappy": {
+    "debian:10": "libsnappy-dev",
+    "ubuntu:focal": "libsnappy-dev",
+    "ubuntu:focal::arch64": "libsnappy-dev",
+    "centos:7": "snappy-devel"
+  },
+  "zlib": {
+    "debian:10": [
+      "libzstd-dev",
+      "zlib1g-dev"
+    ],
+    "ubuntu:focal": [
+      "libzstd-dev",
+      "zlib1g-dev"
+    ],
+    "ubuntu:focal::arch64": [
+      "libzstd-dev",
+      "zlib1g-dev"
+    ],
+    "centos:7": [
+      "zlib-devel",
+      "lz4-devel"
+    ],
+    "centos:8": [
+      "zlib-devel",
+      "lz4-devel"
+    ]
+  },
+  "locales": {
+    "debian:10": "locales",
+    "ubuntu:focal": "locales",
+    "ubuntu:focal::arch64": "locales"
+  },
+  "libtirpc-devel": {
+    "centos:7": "libtirpc-devel",
+    "centos:8": "libtirpc-devel"
+  },
+  "libpmem": {
+    "centos:7": "libpmem-devel"
+  },
+  "make": {
+    "debian:10": "make",
+    "ubuntu:focal": "make",
+    "ubuntu:focal::arch64": "make",
+    "centos:7": "make",
+    "centos:8": "make"
+  },
+  "maven": {
+    "debian:10": "maven",
+    "ubuntu:focal": "maven",
+    "ubuntu:focal::arch64": "maven"
+  },
+  "java": {
+    "debian:10": "openjdk-11-jdk",
+    "ubuntu:focal": [
+      "openjdk-8-jdk",
+      "openjdk-11-jdk"
+    ],
+    "ubuntu:focal::arch64": [
+      "openjdk-8-jdk",
+      "openjdk-11-jdk"
+    ]
+  },
+  "pinentry-curses": {
+    "debian:10": "pinentry-curses",
+    "ubuntu:focal": "pinentry-curses",
+    "ubuntu:focal::arch64": "pinentry-curses",
+    "centos:7": "pinentry-curses",
+    "centos:8": "pinentry-curses"
+  },
+  "pkg-config": {
+    "debian:10": "pkg-config",
+    "ubuntu:focal": "pkg-config",
+    "ubuntu:focal::arch64": "pkg-config",
+    "centos:8": "pkg-config"
+  },
+  "python": {
+    "debian:10": [
+      "python3",
+      "python3-pip",
+      "python3-pkg-resources",
+      "python3-setuptools",
+      "python3-wheel"
+    ],
+    "ubuntu:focal": [
+      "python3",
+      "python3-pip",
+      "python3-pkg-resources",
+      "python3-setuptools",
+      "python3-wheel"
+    ],
+    "ubuntu:focal::arch64": [
+      "python2.7",
+      "python3",
+      "python3-pip",
+      "python3-pkg-resources",
+      "python3-setuptools",
+      "python3-wheel"
+    ],
+    "centos:7": [
+      "python2.7",
+      "python-pip",
+      "python-pkg-resources",
+      "python-setuptools",
+      "python-wheel"
+    ],
+    "centos:8": [
+      "python3",
+      "python3-pip",
+      "python3-setuptools",
+      "python3-wheel"
+    ]
+  },
+  "rsync": {
+    "debian:10": "rsync",
+    "ubuntu:focal": "rsync",
+    "ubuntu:focal::arch64": "rsync",
+    "centos:7": "rsync",
+    "centos:8": "rsync"
+  },
+  "shellcheck": {
+    "debian:10": "shellcheck",
+    "ubuntu:focal": "shellcheck",
+    "ubuntu:focal::arch64": "shellcheck"
+  },
+  "shasum": {
+    "centos:7": "perl-Digest-SHA",
+    "centos:8": "perl-Digest-SHA"
+  },
+  "software-properties-common": {
+    "debian:10": "software-properties-common",
+    "ubuntu:focal": "software-properties-common",
+    "ubuntu:focal::arch64": "software-properties-common"
+  },
+  "sudo": {
+    "debian:10": "sudo",
+    "ubuntu:focal": "sudo",
+    "ubuntu:focal::arch64": "sudo",
+    "centos:7": "sudo",
+    "centos:8": "sudo"
+  },
+  "valgrind": {
+    "debian:10": "valgrind",
+    "ubuntu:focal": "valgrind",
+    "ubuntu:focal::arch64": "valgrind",
+    "centos:7": "valgrind",
+    "centos:8": "valgrind"
+  },
+  "yasm": {
+    "debian:10": "yasm",
+    "ubuntu:focal": "yasm",
+    "ubuntu:focal::arch64": "yasm"
+  }
+}

dev-support/docker/pkg-resolver/platforms.json

@@ -0,0 +1,7 @@
+[
+  "ubuntu:focal",
+  "ubuntu:focal::arch64",
+  "centos:7",
+  "centos:8",
+  "debian:10"
+]

dev-support/docker/pkg-resolver/resolve.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Platform package dependency resolver for building Apache Hadoop.
+"""
+
+import json
+import os
+import sys
+from check_platform import is_supported_platform
+
+
+def get_packages(platform):
+    """
+    Resolve and get the list of packages to install for the given platform.
+
+    :param platform: The platform for which the packages needs to be resolved.
+    :return: A list of resolved packages to install.
+    """
+    with open('pkg-resolver/packages.json', mode='rb') as pkg_file:
+        pkgs = json.loads(pkg_file.read().decode("UTF-8"))
+    packages = []
+    for platforms in [x for x in pkgs.values() if x.get(platform) is not None]:
+        if isinstance(platforms.get(platform), list):
+            packages.extend(platforms.get(platform))
+        else:
+            packages.append(platforms.get(platform))
+    return packages
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        sys.stderr.write('ERROR: Need at least 1 argument, {} were provided{}'.format(
+            len(sys.argv) - 1, os.linesep))
+        sys.exit(1)
+
+    PLATFORM_ARG = sys.argv[1]
+    if not is_supported_platform(PLATFORM_ARG):
+        sys.stderr.write(
+            'ERROR: The given platform {} is not supported. '
+            'Please refer to platforms.json for a list of supported platforms{}'.format(
+                PLATFORM_ARG, os.linesep))
+        sys.exit(1)
+
+    print ' '.join(get_packages(PLATFORM_ARG))