HADOOP-17727. Modularize docker images (#3043)

Reviewed-by: Inigo Goiri <inigoiri@apache.org>

dev-support/bin/create-release

@@ -514,7 +514,7 @@ function dockermode
 
     echo "USER ${user_name}"
     printf "\n\n"
-  ) | docker build -t "${imgname}" -
+  ) | docker build -t "${imgname}" -f - "${BASEDIR}"/dev-support/docker/
 
   run docker run -i -t \
     --privileged \

dev-support/docker/Dockerfile

@@ -32,56 +32,20 @@ RUN echo APT::Install-Suggests "0"\; >>  /etc/apt/apt.conf.d/10disableextras
 ENV DEBIAN_FRONTEND noninteractive
 ENV DEBCONF_TERSE true
 
-# hadolint ignore=DL3008
+######
+# Platform package dependency resolver
+######
+COPY pkg-resolver pkg-resolver
+RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
+    && chmod a+r pkg-resolver/*.json
+
+######
+# Install packages from apt
+######
+# hadolint ignore=DL3008,SC2046
 RUN apt-get -q update \
-    && apt-get -q install -y --no-install-recommends \
-        ant \
-        apt-utils \
-        bats \
-        build-essential \
-        bzip2 \
-        clang \
-        cmake \
-        curl \
-        doxygen \
-        fuse \
-        g++ \
-        gcc \
-        git \
-        gnupg-agent \
-        hugo \
-        libbcprov-java \
-        libbz2-dev \
-        libcurl4-openssl-dev \
-        libfuse-dev \
-        libprotobuf-dev \
-        libprotoc-dev \
-        libsasl2-dev \
-        libsnappy-dev \
-        libssl-dev \
-        libtool \
-        libzstd-dev \
-        locales \
-        make \
-        maven \
-        nodejs \
-        node-yarn \
-        npm \
-        openjdk-11-jdk \
-        openjdk-8-jdk \
-        pinentry-curses \
-        pkg-config \
-        python3 \
-        python3-pip \
-        python3-pkg-resources \
-        python3-setuptools \
-        python3-wheel \
-        rsync \
-        shellcheck \
-        software-properties-common \
-        sudo \
-        valgrind \
-        zlib1g-dev \
+    && apt-get -q install -y --no-install-recommends python3 \
+    && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal) \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
@@ -97,89 +61,16 @@ ENV MAVEN_HOME /usr
 ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
 
 #######
-# Install SpotBugs 4.2.2
+# Set env vars for SpotBugs 4.2.2
 #######
-RUN mkdir -p /opt/spotbugs \
-    && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \
-      -o /opt/spotbugs.tgz \
-    && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \
-    && chmod +x /opt/spotbugs/bin/*
 ENV SPOTBUGS_HOME /opt/spotbugs
 
 #######
-# Install Boost 1.72 (1.71 ships with Focal)
+# Set env vars for Google Protobuf 3.7.1
 #######
-# hadolint ignore=DL3003
-RUN mkdir -p /opt/boost-library \
-    && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \
-    && mv boost_1_72_0.tar.bz2 /opt/boost-library \
-    && cd /opt/boost-library \
-    && tar --bzip2 -xf boost_1_72_0.tar.bz2 \
-    && cd /opt/boost-library/boost_1_72_0 \
-    && ./bootstrap.sh --prefix=/usr/ \
-    && ./b2 --without-python install \
-    && cd /root \
-    && rm -rf /opt/boost-library
-
-######
-# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal)
-######
-# hadolint ignore=DL3003
-RUN mkdir -p /opt/protobuf-src \
-    && curl -L -s -S \
-      https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
-      -o /opt/protobuf.tar.gz \
-    && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
-    && cd /opt/protobuf-src \
-    && ./configure --prefix=/opt/protobuf \
-    && make "-j$(nproc)" \
-    && make install \
-    && cd /root \
-    && rm -rf /opt/protobuf-src
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"
 
-####
-# Install pylint and python-dateutil
-####
-RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1
-
-####
-# Install bower
-####
-# hadolint ignore=DL3008
-RUN npm install -g bower@1.8.8
-
-###
-# Install hadolint
-####
-RUN curl -L -s -S \
-        https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \
-        -o /bin/hadolint \
-   && chmod a+rx /bin/hadolint \
-   && shasum -a 512 /bin/hadolint | \
-        awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}'
-
-######
-# Intel ISA-L 2.29.0
-######
-# hadolint ignore=DL3003,DL3008
-RUN mkdir -p /opt/isa-l-src \
-    && apt-get -q update \
-    && apt-get install -y --no-install-recommends automake yasm \
-    && apt-get clean \
-    && curl -L -s -S \
-      https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \
-      -o /opt/isa-l.tar.gz \
-    && tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src \
-    && cd /opt/isa-l-src \
-    && ./autogen.sh \
-    && ./configure \
-    && make "-j$(nproc)" \
-    && make install \
-    && cd /root \
-    && rm -rf /opt/isa-l-src
-
 ###
 # Avoid out of memory errors in builds
 ###
@@ -188,6 +79,16 @@ ENV MAVEN_OPTS -Xms256m -Xmx3072m
 # Skip gpg verification when downloading Yetus via yetus-wrapper
 ENV HADOOP_SKIP_YETUS_VERIFICATION true
 
+####
+# Install packages
+####
+RUN pkg-resolver/install-common-pkgs.sh
+RUN pkg-resolver/install-spotbugs.sh ubuntu:focal
+RUN pkg-resolver/install-boost.sh ubuntu:focal
+RUN pkg-resolver/install-protobuf.sh ubuntu:focal
+RUN pkg-resolver/install-hadolint.sh ubuntu:focal
+RUN pkg-resolver/install-intel-isa-l.sh ubuntu:focal
+
 ###
 # Everything past this point is either not needed for testing or breaks Yetus.
 # So tell Yetus not to read the rest of the file:

dev-support/docker/Dockerfile_aarch64

@@ -33,61 +33,19 @@ ENV DEBIAN_FRONTEND noninteractive
 ENV DEBCONF_TERSE true
 
 ######
-# Install common dependencies from packages. Versions here are either
-# sufficient or irrelevant.
+# Platform package dependency resolver
 ######
-# hadolint ignore=DL3008
+COPY pkg-resolver pkg-resolver
+RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
+    && chmod a+r pkg-resolver/*.json
+
+######
+# Install packages from apt
+######
+# hadolint ignore=DL3008,SC2046
 RUN apt-get -q update \
-    && apt-get -q install -y --no-install-recommends \
-        ant \
-        apt-utils \
-        bats \
-        build-essential \
-        bzip2 \
-        clang \
-        cmake \
-        curl \
-        doxygen \
-        fuse \
-        g++ \
-        gcc \
-        git \
-        gnupg-agent \
-        hugo \
-        libbcprov-java \
-        libbz2-dev \
-        libcurl4-openssl-dev \
-        libfuse-dev \
-        libprotobuf-dev \
-        libprotoc-dev \
-        libsasl2-dev \
-        libsnappy-dev \
-        libssl-dev \
-        libtool \
-        libzstd-dev \
-        locales \
-        make \
-        maven \
-        nodejs \
-        node-yarn \
-        npm \
-        openjdk-11-jdk \
-        openjdk-8-jdk \
-        phantomjs \
-        pinentry-curses \
-        pkg-config \
-        python2.7 \
-        python3 \
-        python3-pip \
-        python3-pkg-resources \
-        python3-setuptools \
-        python3-wheel \
-        rsync \
-        shellcheck \
-        software-properties-common \
-        sudo \
-        valgrind \
-        zlib1g-dev \
+    && apt-get -q install -y --no-install-recommends python3 \
+    && apt-get -q install -y --no-install-recommends $(pkg-resolver/resolve.py ubuntu:focal::arch64) \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
@@ -103,59 +61,16 @@ ENV MAVEN_HOME /usr
 ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-arm64
 
 #######
-# Install SpotBugs 4.2.2
+# Set env vars for SpotBugs 4.2.2
 #######
-RUN mkdir -p /opt/spotbugs \
-    && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \
-      -o /opt/spotbugs.tgz \
-    && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \
-    && chmod +x /opt/spotbugs/bin/*
 ENV SPOTBUGS_HOME /opt/spotbugs
 
 #######
-# Install Boost 1.72 (1.71 ships with Focal)
+# Set env vars for Google Protobuf 3.7.1
 #######
-# hadolint ignore=DL3003
-RUN mkdir -p /opt/boost-library \
-    && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \
-    && mv boost_1_72_0.tar.bz2 /opt/boost-library \
-    && cd /opt/boost-library \
-    && tar --bzip2 -xf boost_1_72_0.tar.bz2 \
-    && cd /opt/boost-library/boost_1_72_0 \
-    && ./bootstrap.sh --prefix=/usr/ \
-    && ./b2 --without-python install \
-    && cd /root \
-    && rm -rf /opt/boost-library
-
-######
-# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal)
-######
-# hadolint ignore=DL3003
-RUN mkdir -p /opt/protobuf-src \
-    && curl -L -s -S \
-      https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
-      -o /opt/protobuf.tar.gz \
-    && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
-    && cd /opt/protobuf-src \
-    && ./configure --prefix=/opt/protobuf \
-    && make "-j$(nproc)" \
-    && make install \
-    && cd /root \
-    && rm -rf /opt/protobuf-src
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"
 
-####
-# Install pylint and python-dateutil
-####
-RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1
-
-####
-# Install bower
-####
-# hadolint ignore=DL3008
-RUN npm install -g bower@1.8.8
-
 ###
 # Avoid out of memory errors in builds
 ###
@@ -167,6 +82,14 @@ ENV HADOOP_SKIP_YETUS_VERIFICATION true
 # Force PhantomJS to be in 'headless' mode, do not connect to Xwindow
 ENV QT_QPA_PLATFORM offscreen
 
+####
+# Install packages
+####
+RUN pkg-resolver/install-common-pkgs.sh
+RUN pkg-resolver/install-spotbugs.sh ubuntu:focal::arch64
+RUN pkg-resolver/install-boost.sh ubuntu:focal::arch64
+RUN pkg-resolver/install-protobuf.sh ubuntu:focal::arch64
+
 ###
 # Everything past this point is either not needed for testing or breaks Yetus.
 # So tell Yetus not to read the rest of the file:

dev-support/docker/Dockerfile_centos_7

@@ -23,79 +23,28 @@ WORKDIR /root
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
+######
+# Platform package dependency resolver
+######
+COPY pkg-resolver pkg-resolver
+RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
+    && chmod a+r pkg-resolver/*.json
+
+######
+# Install packages from yum
+######
+# hadolint ignore=DL3008,SC2046
 RUN yum update -y \
-    && yum install -y centos-release-scl \
-    && yum install -y devtoolset-9 \
     && yum install -y \
-    ant \
-    build-essential \
-    bzip2 \
-    bzip2-devel \
-    clang \
-    curl \
-    cyrus-sasl-devel \
-    doxygen \
-    fuse \
-    fuse-libs \
-    fuse-devel \
-    git \
-    libcurl-devel \
-    libtirpc-devel \
-    libpmem-devel \
-    libtool \
-    lz4-devel \
-    make \
-    openssl-devel \
-    pinentry-curses \
-    python3 \
-    python3-pip \
-    python3-setuptools \
-    python3-wheel \
-    rsync \
-    snappy-devel \
-    sudo \
-    valgrind \
-    zlib-devel
+        centos-release-scl \
+        python3 \
+    && yum install -y $(pkg-resolver/resolve.py centos:7)
 
 # Set GCC 9 as the default C/C++ compiler
 RUN echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc
 SHELL ["/bin/bash", "--login", "-c"]
 
-####
-# Install Maven 3.6.3
-####
-RUN mkdir -p /opt/maven /tmp/maven \
-    && curl -L -s -S https://mirrors.estointernet.in/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \
-        -o /tmp/maven/apache-maven-3.6.3-bin.tar.gz \
-    && tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven
-
-####
-# Install CMake 3.19
-####
-# hadolint ignore=DL3003
-RUN mkdir -p /tmp/cmake /opt/cmake \
-    && curl -L -s -S https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz -o /tmp/cmake/cmake-3.19.0.tar.gz \
-    && tar xzf /tmp/cmake/cmake-3.19.0.tar.gz --strip-components 1 -C /opt/cmake \
-    && cd /opt/cmake || exit && ./bootstrap \
-    && make "-j$(nproc)" \
-    && make install \
-    && cd /root || exit
-
-####
-# Install zstandard
-####
-# hadolint ignore=DL3003
-RUN mkdir -p /opt/zstd /tmp/zstd \
-    && curl -L -s -S https://github.com/facebook/zstd/archive/refs/tags/v1.4.9.tar.gz -o /tmp/zstd/v1.4.9.tar.gz \
-    && tar xzf /tmp/zstd/v1.4.9.tar.gz --strip-components 1 -C /opt/zstd \
-    && cd /opt/zstd || exit \
-    && make "-j$(nproc)" \
-    && make install \
-    && cd /root || exit
-
-RUN locale-gen en_US.UTF-8
-ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
-ENV PYTHONIOENCODING=utf-8
+# TODO: Set locale
 
 ######
 # Set env vars required to build Hadoop
@@ -106,68 +55,26 @@ ENV PATH "${PATH}:${MAVEN_HOME}/bin"
 ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
 
 #######
-# Install SpotBugs 4.2.2
+# Set env vars for SpotBugs
 #######
-RUN mkdir -p /opt/spotbugs \
-    && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \
-      -o /opt/spotbugs.tgz \
-    && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \
-    && chmod +x /opt/spotbugs/bin/*
 ENV SPOTBUGS_HOME /opt/spotbugs
 
 #######
-# Install Boost 1.72 (1.71 ships with Focal)
+# Set env vars for Google Protobuf
 #######
-# hadolint ignore=DL3003
-RUN mkdir -p /opt/boost-library \
-    && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \
-    && mv boost_1_72_0.tar.bz2 /opt/boost-library \
-    && cd /opt/boost-library \
-    && tar --bzip2 -xf boost_1_72_0.tar.bz2 \
-    && cd /opt/boost-library/boost_1_72_0 \
-    && ./bootstrap.sh --prefix=/usr/ \
-    && ./b2 --without-python install \
-    && cd /root \
-    && rm -rf /opt/boost-library
-
-######
-# Install Google Protobuf 3.7.1 (3.6.1 ships with Focal)
-######
-# hadolint ignore=DL3003
-RUN mkdir -p /opt/protobuf-src \
-    && curl -L -s -S \
-      https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
-      -o /opt/protobuf.tar.gz \
-    && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
-    && cd /opt/protobuf-src \
-    && ./configure --prefix=/opt/protobuf \
-    && make "-j$(nproc)" \
-    && make install \
-    && cd /root \
-    && rm -rf /opt/protobuf-src
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"
 
-####
-# Install Node.js
-####
-# hadolint ignore=DL3003
-RUN mkdir -p /tmp/node \
-    && curl -L -s -S https://nodejs.org/dist/v14.16.1/node-v14.16.1.tar.gz -o /tmp/node-v14.16.1.tar.gz \
-    && tar xzf /tmp/node-v14.16.1.tar.gz --strip-components 1 -C /tmp/node \
-    && cd /tmp/node || exit \
-    && ./configure \
-    && make "-j$(nproc)" \
-    && make install \
-    && cd /root || exit
 
-####
-# Install pylint and python-dateutil
-####
-RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1
-
-####
-# Install bower
-####
-# hadolint ignore=DL3008
-RUN npm install -g bower@1.8.8
+######
+# Install packages
+######
+RUN pkg-resolver/install-maven.sh centos:7
+RUN pkg-resolver/install-cmake.sh centos:7
+RUN pkg-resolver/install-zstandard.sh centos:7
+RUN pkg-resolver/install-yasm.sh centos:7
+RUN pkg-resolver/install-protobuf.sh centos:7
+RUN pkg-resolver/install-boost.sh centos:7
+RUN pkg-resolver/install-spotbugs.sh centos:7
+RUN pkg-resolver/install-nodejs.sh centos:7
+RUN pkg-resolver/install-common-pkgs.sh

dev-support/docker/Dockerfile_centos_8

@@ -23,48 +23,30 @@ WORKDIR /root
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
+######
+# Platform package dependency resolver
+######
+COPY pkg-resolver pkg-resolver
+RUN chmod a+x pkg-resolver/*.sh pkg-resolver/*.py \
+    && chmod a+r pkg-resolver/*.json
+
+######
+# Install packages from yum
+######
+# hadolint ignore=DL3008,SC2046
 RUN yum update -y \
-    && yum install -y \
-        ant \
-        bzip2 \
-        bzip2-devel \
-        clang \
-        curl \
-        cyrus-sasl-devel \
-        dnf \
-        fuse \
-        git \
-        libcurl-devel \
-        fuse \
-        fuse-libs \
-        fuse-devel \
-        libtool \
-        libtirpc-devel \
-        lz4-devel \
-        make \
-        openssl-devel \
-        pinentry-curses \
-        pkg-config \
-        python3 \
-        python3-pip \
-        python3-setuptools \
-        python3-wheel \
-        rsync \
-        sudo \
-        valgrind \
-        zlib-devel
+    && yum install -y python3 \
+    && yum install -y $(pkg-resolver/resolve.py centos:8)
 
 ####
 # Install EPEL
 ####
-RUN mkdir -p /tmp/epel \
-    && curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \
-        -o /tmp/epel/epel-release-latest-8.noarch.rpm \
-    && rpm -Uvh /tmp/epel/epel-release-latest-8.noarch.rpm
+RUN pkg-resolver/install-epel.sh centos:8
 
 RUN dnf --enablerepo=powertools install -y \
     doxygen \
-    snappy-devel
+    snappy-devel \
+    yasm
 
 RUN dnf install -y \
     bouncycastle \
@@ -78,100 +60,34 @@ RUN dnf install -y \
 RUN echo "source /opt/rh/gcc-toolset-9/enable" >> /etc/bashrc
 SHELL ["/bin/bash", "--login", "-c"]
 
-####
-# Install Maven 3.6.3
-####
-RUN mkdir -p /opt/maven /tmp/maven \
-    && curl -L -s -S https://mirrors.estointernet.in/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \
-        -o /tmp/maven/apache-maven-3.6.3-bin.tar.gz \
-    && tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven
-
-####
-# Install CMake 3.19
-####
-# hadolint ignore=DL3003
-RUN mkdir -p /tmp/cmake /opt/cmake \
-    && curl -L -s -S https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz -o /tmp/cmake/cmake-3.19.0.tar.gz \
-    && tar xzf /tmp/cmake/cmake-3.19.0.tar.gz --strip-components 1 -C /opt/cmake \
-    && cd /opt/cmake || exit && ./bootstrap \
-    && make "-j$(nproc)" \
-    && make install \
-    && cd /root || exit
-
-####
-# Install zstandard
-####
-# hadolint ignore=DL3003
-RUN mkdir -p /opt/zstd /tmp/zstd \
-    && curl -L -s -S https://github.com/facebook/zstd/archive/refs/tags/v1.4.9.tar.gz -o /tmp/zstd/v1.4.9.tar.gz \
-    && tar xzf /tmp/zstd/v1.4.9.tar.gz --strip-components 1 -C /opt/zstd \
-    && cd /opt/zstd || exit \
-    && make "-j$(nproc)" \
-    && make install \
-    && cd /root || exit
-
-RUN locale-gen en_US.UTF-8
-ENV LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
-ENV PYTHONIOENCODING=utf-8
+# TODO: Set locale
 
 ######
 # Set env vars required to build Hadoop
 ######
-ENV MAVEN_HOME /opt/maven/apache-maven-3.6.3
+ENV MAVEN_HOME /opt/maven
 ENV PATH "${PATH}:${MAVEN_HOME}/bin"
 # JAVA_HOME must be set in Maven >= 3.5.0 (MNG-6003)
 ENV JAVA_HOME /usr/lib/jvm/java-1.8.0
 
 #######
-# Install SpotBugs 4.2.2
+# Set env vars for SpotBugs
 #######
-RUN mkdir -p /opt/spotbugs \
-    && curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \
-      -o /opt/spotbugs.tgz \
-    && tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs \
-    && chmod +x /opt/spotbugs/bin/*
 ENV SPOTBUGS_HOME /opt/spotbugs
 
 #######
-# Install Boost 1.72
+# Set env vars for Google Protobuf
 #######
-# hadolint ignore=DL3003
-RUN mkdir -p /opt/boost-library \
-    && curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download > boost_1_72_0.tar.bz2 \
-    && mv boost_1_72_0.tar.bz2 /opt/boost-library \
-    && cd /opt/boost-library \
-    && tar --bzip2 -xf boost_1_72_0.tar.bz2 \
-    && cd /opt/boost-library/boost_1_72_0 \
-    && ./bootstrap.sh --prefix=/usr/ \
-    && ./b2 --without-python install \
-    && cd /root \
-    && rm -rf /opt/boost-library
-
-######
-# Install Google Protobuf 3.7.1
-######
-# hadolint ignore=DL3003
-RUN mkdir -p /opt/protobuf-src \
-    && curl -L -s -S \
-      https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
-      -o /opt/protobuf.tar.gz \
-    && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
-    && cd /opt/protobuf-src \
-    && ./configure --prefix=/opt/protobuf \
-    && make "-j$(nproc)" \
-    && make install \
-    && cd /root \
-    && rm -rf /opt/protobuf-src
 ENV PROTOBUF_HOME /opt/protobuf
 ENV PATH "${PATH}:/opt/protobuf/bin"
 
-####
-# Install pylint and python-dateutil
-####
-RUN pip3 install pylint==2.6.0 python-dateutil==2.8.1
-
-####
-# Install bower
-####
-# hadolint ignore=DL3008
-RUN npm install -g bower@1.8.8
+######
+# Install packages
+######
+RUN pkg-resolver/install-maven.sh centos:8
+RUN pkg-resolver/install-cmake.sh centos:8
+RUN pkg-resolver/install-boost.sh centos:8
+RUN pkg-resolver/install-spotbugs.sh centos:8
+RUN pkg-resolver/install-protobuf.sh centos:8
+RUN pkg-resolver/install-zstandard.sh centos:8
+RUN pkg-resolver/install-common-pkgs.sh

dev-support/docker/README.md

@@ -0,0 +1,74 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+
+# Docker images for building Hadoop
+
+This folder contains the Dockerfiles for building Hadoop on various platforms.
+
+# Dependency management
+
+The mode of installation of the dependencies needed for building Hadoop varies from one platform to
+the other. Different platforms have different toolchains. Some packages tend to be polymorphic
+across platforms and most commonly, a package that's readily available in one platform's toolchain
+isn't available on another. We thus, resort to building and installing the package from source,
+causing duplication of code since this needs to be done for all the Dockerfiles pertaining to all
+the platforms. We need a system to track a dependency - for a package - for a platform. Thus,
+there's a lot of diversity that needs to be handled for managing package dependencies and
+`pkg-resolver` caters to that.
+
+## Supported platforms
+
+`pkg-resolver/platforms.json` contains a list of the supported platforms for dependency management.
+
+## Package dependencies
+
+`pkg-resolver/packages.json` maps a dependency to a given platform. Here's the schema of this JSON.
+
+```json
+{
+  "dependency_1": {
+    "platform_1": "package_1",
+    "platform_2": [
+      "package_1",
+      "package_2"
+    ]
+  },
+  "dependency_2": {
+    "platform_1": [
+      "package_1",
+      "package_2",
+      "package_3"
+    ]
+  }
+}
+```
+
+The root JSON element contains unique _dependency_ children. This in turn contains the name of the _
+platforms_ and the list of _packages_ to be installed for that platform. Just to give an example of
+how to interpret the above JSON -
+
+1. For `dependency_1`, `package_1` needs to be installed for `platform_1`.
+2. For `dependency_2`, `package_1` and `package_2` needs to be installed for `platform_2`.
+3. For `dependency_2`, `package_1`, `package_3` and `package_3` needs to be installed for
+   `platform_1`.
+
+## Standalone packages
+
+Most commonly, some packages are not available across the toolchains in various platforms. Thus, we
+would need to build and install them. Since we need to do this across all the Dockerfiles for all
+the platforms, it could lead to code duplication and managing them becomes a hassle. Thus, we put
+the build steps in a `pkg-resolver/install-<package>.sh` and invoke this in all the Dockerfiles.

dev-support/docker/pkg-resolver/check_platform.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Checks whether the given platform is supported for building Apache Hadoop
+"""
+
+import json
+import sys
+
+
+def get_platforms():
+    """
+    :return: A list of the supported platforms managed by pkg-resolver.
+    """
+
+    with open('pkg-resolver/platforms.json', encoding='utf-8', mode='r') as platforms_file:
+        return json.loads(platforms_file.read())
+
+
+def is_supported_platform(platform):
+    """
+    :param platform: The name of the platform
+    :return: Whether the platform is supported
+    """
+    return platform in get_platforms()
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+        print('ERROR: Expecting 1 argument, {} were provided'.format(len(sys.argv) - 1),
+              file=sys.stderr)
+        sys.exit(1)
+
+    sys.exit(0 if is_supported_platform(sys.argv[1]) else 1)

dev-support/docker/pkg-resolver/install-boost.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ]; then
+  echo "ERROR: Need at least 1 argument, $# were provided"
+  exit 1
+fi
+
+pkg-resolver/check_platform.py "$1"
+if [ $? -eq 1 ]; then
+  echo "ERROR: Unsupported platform $1"
+  exit 1
+fi
+
+default_version="1.72.0"
+version_to_install=$default_version
+if [ -n "$2" ]; then
+  version_to_install="$2"
+fi
+
+if [ "$version_to_install" != "1.72.0" ]; then
+  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
+  version_to_install=$default_version
+fi
+
+if [ "$version_to_install" == "1.72.0" ]; then
+  # hadolint ignore=DL3003
+  mkdir -p /opt/boost-library &&
+    curl -L https://sourceforge.net/projects/boost/files/boost/1.72.0/boost_1_72_0.tar.bz2/download >boost_1_72_0.tar.bz2 &&
+    mv boost_1_72_0.tar.bz2 /opt/boost-library &&
+    cd /opt/boost-library &&
+    tar --bzip2 -xf boost_1_72_0.tar.bz2 &&
+    cd /opt/boost-library/boost_1_72_0 &&
+    ./bootstrap.sh --prefix=/usr/ &&
+    ./b2 --without-python install &&
+    cd /root &&
+    rm -rf /opt/boost-library
+else
+  echo "ERROR: Don't know how to install version $version_to_install"
+  exit 1
+fi

dev-support/docker/pkg-resolver/install-cmake.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ]; then
+  echo "ERROR: Need at least 1 argument, $# were provided"
+  exit 1
+fi
+
+pkg-resolver/check_platform.py "$1"
+if [ $? -eq 1 ]; then
+  echo "ERROR: Unsupported platform $1"
+  exit 1
+fi
+
+default_version="3.19.0"
+version_to_install=$default_version
+if [ -n "$2" ]; then
+  version_to_install="$2"
+fi
+
+if [ "$version_to_install" != "3.19.0" ]; then
+  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
+  version_to_install=$default_version
+fi
+
+if [ "$version_to_install" == "3.19.0" ]; then
+  # hadolint ignore=DL3003
+  mkdir -p /tmp/cmake /opt/cmake &&
+    curl -L -s -S https://cmake.org/files/v3.19/cmake-3.19.0.tar.gz -o /tmp/cmake/cmake-3.19.0.tar.gz &&
+    tar xzf /tmp/cmake/cmake-3.19.0.tar.gz --strip-components 1 -C /opt/cmake &&
+    cd /opt/cmake || exit && ./bootstrap &&
+    make "-j$(nproc)" &&
+    make install &&
+    cd /root || exit
+else
+  echo "ERROR: Don't know how to install version $version_to_install"
+  exit 1
+fi

dev-support/docker/pkg-resolver/install-common-pkgs.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+######
+# Install pylint and python-dateutil
+######
+pip3 install pylint==2.6.0 python-dateutil==2.8.1
+
+######
+# Install bower
+######
+# hadolint ignore=DL3008
+npm install -g bower@1.8.8

dev-support/docker/pkg-resolver/install-epel.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ]; then
+  echo "ERROR: Need at least 1 argument, $# were provided"
+  exit 1
+fi
+
+pkg-resolver/check_platform.py "$1"
+if [ $? -eq 1 ]; then
+  echo "ERROR: Unsupported platform $1"
+  exit 1
+fi
+
+default_version="8"
+version_to_install=$default_version
+if [ -n "$2" ]; then
+  version_to_install="$2"
+fi
+
+if [ "$version_to_install" != "8" ]; then
+  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
+  version_to_install=$default_version
+fi
+
+if [ "$version_to_install" == "8" ]; then
+  mkdir -p /tmp/epel &&
+    curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm \
+      -o /tmp/epel/epel-release-latest-8.noarch.rpm &&
+    rpm -Uvh /tmp/epel/epel-release-latest-8.noarch.rpm
+else
+  echo "ERROR: Don't know how to install version $version_to_install"
+  exit 1
+fi

dev-support/docker/pkg-resolver/install-hadolint.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ]; then
+  echo "ERROR: Need at least 1 argument, $# were provided"
+  exit 1
+fi
+
+pkg-resolver/check_platform.py "$1"
+if [ $? -eq 1 ]; then
+  echo "ERROR: Unsupported platform $1"
+  exit 1
+fi
+
+curl -L -s -S \
+  https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \
+  -o /bin/hadolint &&
+  chmod a+rx /bin/hadolint &&
+  shasum -a 512 /bin/hadolint |
+  awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}'

dev-support/docker/pkg-resolver/install-intel-isa-l.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ]; then
+  echo "ERROR: Need at least 1 argument, $# were provided"
+  exit 1
+fi
+
+pkg-resolver/check_platform.py "$1"
+if [ $? -eq 1 ]; then
+  echo "ERROR: Unsupported platform $1"
+  exit 1
+fi
+
+default_version="2.29.0"
+version_to_install=$default_version
+if [ -n "$2" ]; then
+  version_to_install="$2"
+fi
+
+if [ "$version_to_install" != "2.29.0" ]; then
+  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
+  version_to_install=$default_version
+fi
+
+if [ "$version_to_install" == "2.29.0" ]; then
+  # hadolint ignore=DL3003,DL3008
+  mkdir -p /opt/isa-l-src &&
+    curl -L -s -S \
+      https://github.com/intel/isa-l/archive/v2.29.0.tar.gz \
+      -o /opt/isa-l.tar.gz &&
+    tar xzf /opt/isa-l.tar.gz --strip-components 1 -C /opt/isa-l-src &&
+    cd /opt/isa-l-src &&
+    ./autogen.sh &&
+    ./configure &&
+    make "-j$(nproc)" &&
+    make install &&
+    cd /root &&
+    rm -rf /opt/isa-l-src
+else
+  echo "ERROR: Don't know how to install version $version_to_install"
+  exit 1
+fi

dev-support/docker/pkg-resolver/install-maven.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ]; then
+  echo "ERROR: Need at least 1 argument, $# were provided"
+  exit 1
+fi
+
+pkg-resolver/check_platform.py "$1"
+if [ $? -eq 1 ]; then
+  echo "ERROR: Unsupported platform $1"
+  exit 1
+fi
+
+default_version="3.6.3"
+version_to_install=$default_version
+if [ -n "$2" ]; then
+  version_to_install="$2"
+fi
+
+if [ "$version_to_install" != "3.6.3" ]; then
+  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
+  version_to_install=$default_version
+fi
+
+if [ "$version_to_install" == "3.6.3" ]; then
+  mkdir -p /opt/maven /tmp/maven &&
+    curl -L -s -S https://mirrors.estointernet.in/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz \
+      -o /tmp/maven/apache-maven-3.6.3-bin.tar.gz &&
+    tar xzf /tmp/maven/apache-maven-3.6.3-bin.tar.gz --strip-components 1 -C /opt/maven
+else
+  echo "ERROR: Don't know how to install version $version_to_install"
+  exit 1
+fi

dev-support/docker/pkg-resolver/install-nodejs.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ]; then
+  echo "ERROR: Need at least 1 argument, $# were provided"
+  exit 1
+fi
+
+pkg-resolver/check_platform.py "$1"
+if [ $? -eq 1 ]; then
+  echo "ERROR: Unsupported platform $1"
+  exit 1
+fi
+
+default_version="14.16.1"
+version_to_install=$default_version
+if [ -n "$2" ]; then
+  version_to_install="$2"
+fi
+
+if [ "$version_to_install" != "14.16.1" ]; then
+  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
+  version_to_install=$default_version
+fi
+
+if [ "$version_to_install" == "14.16.1" ]; then
+  # hadolint ignore=DL3003
+  mkdir -p /tmp/node &&
+    curl -L -s -S https://nodejs.org/dist/v14.16.1/node-v14.16.1.tar.gz -o /tmp/node-v14.16.1.tar.gz &&
+    tar xzf /tmp/node-v14.16.1.tar.gz --strip-components 1 -C /tmp/node &&
+    cd /tmp/node || exit &&
+    ./configure &&
+    make "-j$(nproc)" &&
+    make install &&
+    cd /root || exit
+else
+  echo "ERROR: Don't know how to install version $version_to_install"
+  exit 1
+fi

dev-support/docker/pkg-resolver/install-protobuf.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ]; then
+  echo "ERROR: Need at least 1 argument, $# were provided"
+  exit 1
+fi
+
+pkg-resolver/check_platform.py "$1"
+if [ $? -eq 1 ]; then
+  echo "ERROR: Unsupported platform $1"
+  exit 1
+fi
+
+default_version="3.7.1"
+version_to_install=$default_version
+if [ -n "$2" ]; then
+  version_to_install="$2"
+fi
+
+if [ "$version_to_install" != "3.7.1" ]; then
+  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
+  version_to_install=$default_version
+fi
+
+if [ "$version_to_install" == "3.7.1" ]; then
+  # hadolint ignore=DL3003
+  mkdir -p /opt/protobuf-src &&
+    curl -L -s -S \
+      https://github.com/protocolbuffers/protobuf/releases/download/v3.7.1/protobuf-java-3.7.1.tar.gz \
+      -o /opt/protobuf.tar.gz &&
+    tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src &&
+    cd /opt/protobuf-src &&
+    ./configure --prefix=/opt/protobuf &&
+    make "-j$(nproc)" &&
+    make install &&
+    cd /root &&
+    rm -rf /opt/protobuf-src
+else
+  echo "ERROR: Don't know how to install version $version_to_install"
+  exit 1
+fi

dev-support/docker/pkg-resolver/install-spotbugs.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ]; then
+  echo "ERROR: Need at least 1 argument, $# were provided"
+  exit 1
+fi
+
+pkg-resolver/check_platform.py "$1"
+if [ $? -eq 1 ]; then
+  echo "ERROR: Unsupported platform $1"
+  exit 1
+fi
+
+default_version="4.2.2"
+version_to_install=$default_version
+if [ -n "$2" ]; then
+  version_to_install="$2"
+fi
+
+if [ "$version_to_install" != "4.2.2" ]; then
+  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
+  version_to_install=$default_version
+fi
+
+if [ "$version_to_install" == "4.2.2" ]; then
+  mkdir -p /opt/spotbugs &&
+    curl -L -s -S https://github.com/spotbugs/spotbugs/releases/download/4.2.2/spotbugs-4.2.2.tgz \
+      -o /opt/spotbugs.tgz &&
+    tar xzf /opt/spotbugs.tgz --strip-components 1 -C /opt/spotbugs &&
+    chmod +x /opt/spotbugs/bin/*
+else
+  echo "ERROR: Don't know how to install version $version_to_install"
+  exit 1
+fi

dev-support/docker/pkg-resolver/install-yasm.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ]; then
+  echo "ERROR: Need at least 1 argument, $# were provided"
+  exit 1
+fi
+
+pkg-resolver/check_platform.py "$1"
+if [ $? -eq 1 ]; then
+  echo "ERROR: Unsupported platform $1"
+  exit 1
+fi
+
+default_version="1.2.0-4"
+version_to_install=$default_version
+if [ -n "$2" ]; then
+  version_to_install="$2"
+fi
+
+if [ "$version_to_install" != "1.2.0-4" ]; then
+  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
+  version_to_install=$default_version
+fi
+
+if [ "$version_to_install" == "1.2.0-4" ]; then
+  mkdir -p /tmp/yasm &&
+    curl -L -s -S https://download-ib01.fedoraproject.org/pub/epel/7/x86_64/Packages/y/yasm-1.2.0-4.el7.x86_64.rpm \
+      -o /tmp/yasm-1.2.0-4.el7.x86_64.rpm &&
+    rpm -Uvh /tmp/yasm-1.2.0-4.el7.x86_64.rpm
+else
+  echo "ERROR: Don't know how to install version $version_to_install"
+  exit 1
+fi

dev-support/docker/pkg-resolver/install-zstandard.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ $# -lt 1 ]; then
+  echo "ERROR: Need at least 1 argument, $# were provided"
+  exit 1
+fi
+
+pkg-resolver/check_platform.py "$1"
+if [ $? -eq 1 ]; then
+  echo "ERROR: Unsupported platform $1"
+  exit 1
+fi
+
+default_version="1.4.9"
+version_to_install=$default_version
+if [ -n "$2" ]; then
+  version_to_install="$2"
+fi
+
+if [ "$version_to_install" != "1.4.9" ]; then
+  echo "WARN: Don't know how to install version $version_to_install, installing the default version $default_version instead"
+  version_to_install=$default_version
+fi
+
+if [ "$version_to_install" == "1.4.9" ]; then
+  # hadolint ignore=DL3003
+  mkdir -p /opt/zstd /tmp/zstd &&
+    curl -L -s -S https://github.com/facebook/zstd/archive/refs/tags/v1.4.9.tar.gz -o /tmp/zstd/v1.4.9.tar.gz &&
+    tar xzf /tmp/zstd/v1.4.9.tar.gz --strip-components 1 -C /opt/zstd &&
+    cd /opt/zstd || exit &&
+    make "-j$(nproc)" &&
+    make install &&
+    cd /root || exit
+else
+  echo "ERROR: Don't know how to install version $version_to_install"
+  exit 1
+fi

dev-support/docker/pkg-resolver/packages.json

@@ -0,0 +1,302 @@
+{
+  "ant": {
+    "ubuntu:focal": "ant",
+    "ubuntu:focal::arch64": "ant",
+    "centos:7": "ant",
+    "centos:8": "ant"
+  },
+  "apt-utils": {
+    "ubuntu:focal": "apt-utils",
+    "ubuntu:focal::arch64": "apt-utils"
+  },
+  "automake": {
+    "ubuntu:focal": "automake",
+    "ubuntu:focal::arch64": "automake",
+    "centos:7": "automake",
+    "centos:8": "automake"
+  },
+  "bats": {
+    "ubuntu:focal": "bats",
+    "ubuntu:focal::arch64": "bats"
+  },
+  "build-essential": {
+    "ubuntu:focal": "build-essential",
+    "ubuntu:focal::arch64": "build-essential",
+    "centos:7": "build-essential"
+  },
+  "bzip2": {
+    "ubuntu:focal": [
+      "bzip2",
+      "libbz2-dev"
+    ],
+    "ubuntu:focal::arch64": [
+      "bzip2",
+      "libbz2-dev"
+    ],
+    "centos:7": [
+      "bzip2",
+      "bzip2-devel"
+    ],
+    "centos:8": [
+      "bzip2",
+      "bzip2-devel"
+    ]
+  },
+  "clang": {
+    "ubuntu:focal": "clang",
+    "ubuntu:focal::arch64": "clang",
+    "centos:7": "clang",
+    "centos:8": "clang"
+  },
+  "cmake": {
+    "ubuntu:focal": "cmake",
+    "ubuntu:focal::arch64": "cmake"
+  },
+  "curl": {
+    "ubuntu:focal": [
+      "curl",
+      "libcurl4-openssl-dev"
+    ],
+    "ubuntu:focal::arch64": [
+      "curl",
+      "libcurl4-openssl-dev"
+    ],
+    "centos:7": [
+      "curl",
+      "libcurl-devel"
+    ],
+    "centos:8": [
+      "curl",
+      "libcurl-devel"
+    ]
+  },
+  "doxygen": {
+    "ubuntu:focal": "doxygen",
+    "ubuntu:focal::arch64": "doxygen",
+    "centos:7": "doxygen"
+  },
+  "dnf": {
+    "centos:8": "dnf"
+  },
+  "fuse": {
+    "ubuntu:focal": [
+      "fuse",
+      "libfuse-dev"
+    ],
+    "ubuntu:focal::arch64": [
+      "fuse",
+      "libfuse-dev"
+    ],
+    "centos:7": [
+      "fuse",
+      "fuse-libs",
+      "fuse-devel"
+    ],
+    "centos:8": [
+      "fuse",
+      "fuse-libs",
+      "fuse-devel"
+    ]
+  },
+  "gcc": {
+    "ubuntu:focal": [
+      "gcc",
+      "g++"
+    ],
+    "ubuntu:focal::arch64": [
+      "gcc",
+      "g++"
+    ],
+    "centos:7": [
+      "centos-release-scl",
+      "devtoolset-9"
+    ]
+  },
+  "git": {
+    "ubuntu:focal": "git",
+    "ubuntu:focal::arch64": "git",
+    "centos:7": "git",
+    "centos:8": "git"
+  },
+  "gnupg-agent": {
+    "ubuntu:focal": "gnupg-agent",
+    "ubuntu:focal::arch64": "gnupg-agent"
+  },
+  "hugo": {
+    "ubuntu:focal": "hugo",
+    "ubuntu:focal::arch64": "hugo"
+  },
+  "libbcprov-java": {
+    "ubuntu:focal": "libbcprov-java",
+    "ubuntu:focal::arch64": "libbcprov-java"
+  },
+  "libtool": {
+    "ubuntu:focal": "libtool",
+    "ubuntu:focal::arch64": "libtool",
+    "centos:7": "libtool",
+    "centos:8": "libtool"
+  },
+  "openssl": {
+    "ubuntu:focal": "libssl-dev",
+    "ubuntu:focal::arch64": "libssl-dev",
+    "centos:7": "openssl-devel",
+    "centos:8": "openssl-devel"
+  },
+  "protocol-buffers": {
+    "ubuntu:focal": [
+      "libprotobuf-dev",
+      "libprotoc-dev"
+    ],
+    "ubuntu:focal::arch64": [
+      "libprotobuf-dev",
+      "libprotoc-dev"
+    ]
+  },
+  "sasl": {
+    "ubuntu:focal": "libsasl2-dev",
+    "ubuntu:focal::arch64": "libsasl2-dev",
+    "centos:7": "cyrus-sasl-devel",
+    "centos:8": "cyrus-sasl-devel"
+  },
+  "snappy": {
+    "ubuntu:focal": "libsnappy-dev",
+    "ubuntu:focal::arch64": "libsnappy-dev",
+    "centos:7": "snappy-devel"
+  },
+  "zlib": {
+    "ubuntu:focal": [
+      "libzstd-dev",
+      "zlib1g-dev"
+    ],
+    "ubuntu:focal::arch64": [
+      "libzstd-dev",
+      "zlib1g-dev"
+    ],
+    "centos:7": [
+      "zlib-devel",
+      "lz4-devel"
+    ],
+    "centos:8": [
+      "zlib-devel",
+      "lz4-devel"
+    ]
+  },
+  "locales": {
+    "ubuntu:focal": "locales",
+    "ubuntu:focal::arch64": "locales"
+  },
+  "libtirpc-devel": {
+    "centos:7": "libtirpc-devel",
+    "centos:8": "libtirpc-devel"
+  },
+  "libpmem": {
+    "centos:7": "libpmem-devel"
+  },
+  "make": {
+    "ubuntu:focal": "make",
+    "ubuntu:focal::arch64": "make",
+    "centos:7": "make",
+    "centos:8": "make"
+  },
+  "maven": {
+    "ubuntu:focal": "maven",
+    "ubuntu:focal::arch64": "maven"
+  },
+  "javascript": {
+    "ubuntu:focal": [
+      "nodejs",
+      "node-yarn",
+      "npm"
+    ],
+    "ubuntu:focal::arch64": [
+      "nodejs",
+      "node-yarn",
+      "npm",
+      "phantomjs"
+    ]
+  },
+  "java": {
+    "ubuntu:focal": [
+      "openjdk-8-jdk",
+      "openjdk-11-jdk"
+    ],
+    "ubuntu:focal::arch64": [
+      "openjdk-8-jdk",
+      "openjdk-11-jdk"
+    ]
+  },
+  "pinentry-curses": {
+    "ubuntu:focal": "pinentry-curses",
+    "ubuntu:focal::arch64": "pinentry-curses",
+    "centos:7": "pinentry-curses",
+    "centos:8": "pinentry-curses"
+  },
+  "pkg-config": {
+    "ubuntu:focal": "pkg-config",
+    "ubuntu:focal::arch64": "pkg-config",
+    "centos:8": "pkg-config"
+  },
+  "python": {
+    "ubuntu:focal": [
+      "python3",
+      "python3-pip",
+      "python3-pkg-resources",
+      "python3-setuptools",
+      "python3-wheel"
+    ],
+    "ubuntu:focal::arch64": [
+      "python2.7",
+      "python3",
+      "python3-pip",
+      "python3-pkg-resources",
+      "python3-setuptools",
+      "python3-wheel"
+    ],
+    "centos:7": [
+      "python3",
+      "python3-pip",
+      "python3-setuptools",
+      "python3-wheel"
+    ],
+    "centos:8": [
+      "python3",
+      "python3-pip",
+      "python3-setuptools",
+      "python3-wheel"
+    ]
+  },
+  "rsync": {
+    "ubuntu:focal": "rsync",
+    "ubuntu:focal::arch64": "rsync",
+    "centos:7": "rsync",
+    "centos:8": "rsync"
+  },
+  "shellcheck": {
+    "ubuntu:focal": "shellcheck",
+    "ubuntu:focal::arch64": "shellcheck"
+  },
+  "shasum": {
+    "centos:7": "perl-Digest-SHA",
+    "centos:8": "perl-Digest-SHA"
+  },
+  "software-properties-common": {
+    "ubuntu:focal": "software-properties-common",
+    "ubuntu:focal::arch64": "software-properties-common"
+  },
+  "sudo": {
+    "ubuntu:focal": "sudo",
+    "ubuntu:focal::arch64": "sudo",
+    "centos:7": "sudo",
+    "centos:8": "sudo"
+  },
+  "valgrind": {
+    "ubuntu:focal": "valgrind",
+    "ubuntu:focal::arch64": "valgrind",
+    "centos:7": "valgrind",
+    "centos:8": "valgrind"
+  },
+  "yasm": {
+    "ubuntu:focal": "yasm",
+    "ubuntu:focal::arch64": "yasm"
+  }
+}

dev-support/docker/pkg-resolver/platforms.json

@@ -0,0 +1,6 @@
+[
+  "ubuntu:focal",
+  "ubuntu:focal::arch64",
+  "centos:7",
+  "centos:8"
+]

dev-support/docker/pkg-resolver/resolve.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Platform package dependency resolver for building Apache Hadoop.
+"""
+
+import json
+import sys
+from check_platform import is_supported_platform
+
+
+def get_packages(platform):
+    """
+    Resolve and get the list of packages to install for the given platform.
+
+    :param platform: The platform for which the packages needs to be resolved.
+    :return: A list of resolved packages to install.
+    """
+    with open('pkg-resolver/packages.json', encoding='utf-8', mode='r') as pkg_file:
+        pkgs = json.loads(pkg_file.read())
+    packages = []
+    for platforms in filter(lambda x: x.get(platform) is not None, pkgs.values()):
+        if isinstance(platforms.get(platform), list):
+            packages.extend(platforms.get(platform))
+        else:
+            packages.append(platforms.get(platform))
+    return packages
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print('ERROR: Need at least 1 argument, {} were provided'.format(len(sys.argv) - 1),
+              file=sys.stderr)
+        sys.exit(1)
+
+    platform_arg = sys.argv[1]
+    if not is_supported_platform(platform_arg):
+        print(
+            'ERROR: The given platform {} is not supported. '
+            'Please refer to platforms.json for a list of supported platforms'.format(
+                platform_arg), file=sys.stderr)
+        sys.exit(1)
+
+    packages_to_install = get_packages(platform_arg)
+    print(' '.join(packages_to_install))